diff --git a/actions/ql/lib/qlpack.yml b/actions/ql/lib/qlpack.yml index 612e631c6f8b..268018a37415 100644 --- a/actions/ql/lib/qlpack.yml +++ b/actions/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/actions-all -version: 0.6.1 +version: 0.6.2-dev library: true warnOnImplicitThis: true dependencies: diff --git a/actions/ql/src/qlpack.yml b/actions/ql/src/qlpack.yml index 5e4877698e63..5feedb52c8ce 100644 --- a/actions/ql/src/qlpack.yml +++ b/actions/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/actions-queries -version: 0.6.35 +version: 0.6.36-dev library: false warnOnImplicitThis: true groups: [actions, queries] diff --git a/cpp/ql/lib/change-notes/2026-08-27-bdlbb-blob-models.md b/cpp/ql/lib/change-notes/2026-08-27-bdlbb-blob-models.md new file mode 100644 index 000000000000..ed52e5e091e2 --- /dev/null +++ b/cpp/ql/lib/change-notes/2026-08-27-bdlbb-blob-models.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* Added flow summaries for the BDE `BloombergLP::bdlbb::Blob` segmented byte buffer. diff --git a/cpp/ql/lib/ext/bdlbb.model.yml b/cpp/ql/lib/ext/bdlbb.model.yml new file mode 100644 index 000000000000..e5c50207c464 --- /dev/null +++ b/cpp/ql/lib/ext/bdlbb.model.yml @@ -0,0 +1,18 @@ +# Model of the BDE bdlbb::Blob segmented byte buffer (BloombergLP::bdlbb). +# Lets taint reach a blob's payload bytes, e.g. a message body filled by bmqa::Message::getData. +extensions: + - addsTo: + pack: codeql/cpp-all + extensible: summaryModel + data: # namespace, type, subtypes, name, signature, ext, input, output, kind, provenance + # Accessor chain + - ["BloombergLP::bdlbb", "Blob", true, "buffer", "", "", "Argument[-1]", "ReturnValue[*]", "taint", "manual"] + - ["BloombergLP::bdlbb", "BlobBuffer", true, "data", "", "", "Argument[-1]", "ReturnValue[*]", "taint", "manual"] + - ["BloombergLP::bdlbb", "BlobBuffer", true, "buffer", "", "", "Argument[-1]", "ReturnValue[*]", "taint", "manual"] + # BlobUtil read-out + - ["BloombergLP::bdlbb", "BlobUtil", true, "copy", "(char *,const Blob &,int,int)", "", "Argument[*1]", "Argument[*0]", "taint", "manual"] + - ["BloombergLP::bdlbb", "BlobUtil", true, "getContiguousRangeOrCopy", "", "", "Argument[*1]", "Argument[*0]", "taint", "manual"] + - ["BloombergLP::bdlbb", "BlobUtil", true, "getContiguousRangeOrCopy", "", "", "Argument[*1]", "ReturnValue[*]", "taint", "manual"] + # BlobUtil write-in + - ["BloombergLP::bdlbb", "BlobUtil", true, "copy", "(Blob *,int,const char *,int)", "", "Argument[*2]", "Argument[*0]", "taint", "manual"] + - ["BloombergLP::bdlbb", "BlobUtil", true, "copy", "(Blob *,int,const Blob &,int,int)", "", "Argument[*2]", "Argument[*0]", "taint", "manual"] diff --git a/cpp/ql/lib/qlpack.yml b/cpp/ql/lib/qlpack.yml index c3e40cb63948..5114bf861a36 100644 --- a/cpp/ql/lib/qlpack.yml +++ b/cpp/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/cpp-all -version: 12.1.0 +version: 12.1.1-dev groups: cpp dbscheme: semmlecode.cpp.dbscheme extractor: cpp diff --git a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/FlowSummaryImpl.qll b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/FlowSummaryImpl.qll index 780c802dc8ae..176b95933db8 100644 --- a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/FlowSummaryImpl.qll +++ b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/FlowSummaryImpl.qll @@ -256,7 +256,7 @@ private module Input2 implements Impl::Private::InputSig2 { pragma[nomagic] private predicate hasKindAndEnclosingFunction(Function f, ReturnKind rk, ReturnNode r) { r.getEnclosingCallable().asSourceCallable() = f and - r.getKind() = rk + pragma[only_bind_into](r).getKind() = rk } pragma[nomagic] diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll index bce936552768..03a565ef946d 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll @@ -131,6 +131,15 @@ private predicate qualifierSourceImpl(RelevantNode n, Class c) { ) } +pragma[nomagic] +private predicate hasKindAndEnclosingCallable( + DataFlowPrivate::DataFlowCallable callable, DataFlowPrivate::ReturnKind kind, + DataFlowPrivate::ReturnNode return +) { + return.getEnclosingCallable() = callable and + return.getKind() = kind +} + private module TrackVirtualDispatch { /** * Gets a possible runtime target of `c` using both static call-target @@ -197,11 +206,21 @@ private module TrackVirtualDispatch { ) } + pragma[nomagic] + private predicate hasDispatchWithKind( + DataFlowPrivate::DataFlowCallable callable, DataFlowPrivate::ReturnKind kind, + LocalSourceNode n2 + ) { + exists(DataFlowPrivate::DataFlowCall call | + n2 = DataFlowPrivate::getAnOutNode(call, kind) and + callable = dispatch(call) + ) + } + predicate returnStep(Node n1, LocalSourceNode n2) { - exists(DataFlowPrivate::DataFlowCallable callable, DataFlowPrivate::DataFlowCall call | - n1.(DataFlowPrivate::ReturnNode).getEnclosingCallable() = callable and - callable = dispatch(call) and - n2 = DataFlowPrivate::getAnOutNode(call, n1.(DataFlowPrivate::ReturnNode).getKind()) + exists(DataFlowPrivate::DataFlowCallable callable, DataFlowPrivate::ReturnKind kind | + hasKindAndEnclosingCallable(callable, kind, n1) and + hasDispatchWithKind(callable, kind, n2) ) } diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll index 3a1b42645642..551035c5589e 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll @@ -122,6 +122,47 @@ private module Cached { FlowSummaryImpl::Private::Steps::summaryJumpStep(n1, n2) } + bindingset[store] + pragma[inline_late] + private predicate nodeHasInstructionLate(Node node, StoreInstruction store, int indirectionIndex) { + nodeHasInstruction(node, store, indirectionIndex) + } + + pragma[nomagic] + private predicate storeStepSource( + Operand fieldAddress, int contentIndirectionIndex, Node node, boolean certain + ) { + exists(int indirectionIndex, int numberOfLoads, StoreInstruction store | + nodeHasInstructionLate(node, store, indirectionIndex) and + numberOfLoadsFromOperand(fieldAddress, store.getDestinationAddressOperand(), numberOfLoads, + certain) and + contentIndirectionIndex = 1 + indirectionIndex + numberOfLoads + ) + } + + pragma[nomagic] + private predicate hasFieldAddressAndField(Field f, PostFieldUpdateNode pfu, Operand fieldAddress) { + pfu.getIndirectionIndex() = 1 and + pfu.getUpdatedField() = f and + pfu.getFieldAddress() = fieldAddress + } + + pragma[nomagic] + private predicate hasFieldAndIndirectionIndex(Field f, int indirectionIndex, FieldContent fc) { + fc.getAField() = f and + fc.getIndirectionIndex() = indirectionIndex + } + + pragma[nomagic] + private predicate storeStepTarget( + Operand address, int indirectionIndex, PostFieldUpdateNode pfu, FieldContent fc + ) { + exists(Field f | + hasFieldAddressAndField(f, pfu, address) and + hasFieldAndIndirectionIndex(f, indirectionIndex, fc) + ) + } + /** * Holds if data can flow from `node1` to `node2` via an assignment to `f`. * Thus, `node2` references an object with a field `f` that contains the @@ -132,19 +173,9 @@ private module Cached { */ cached predicate storeStepImpl(Node node1, Content c, Node node2, boolean certain) { - exists( - PostFieldUpdateNode postFieldUpdate, int indirectionIndex1, int numberOfLoads, - StoreInstruction store, FieldContent fc - | - postFieldUpdate = node2 and - fc = c and - nodeHasInstruction(node1, pragma[only_bind_into](store), - pragma[only_bind_into](indirectionIndex1)) and - postFieldUpdate.getIndirectionIndex() = 1 and - numberOfLoadsFromOperand(postFieldUpdate.getFieldAddress(), - store.getDestinationAddressOperand(), numberOfLoads, certain) and - fc.getAField() = postFieldUpdate.getUpdatedField() and - getIndirectionIndexLate(fc) = 1 + indirectionIndex1 + numberOfLoads + exists(Operand fieldAddress, int indirectionIndex | + storeStepSource(fieldAddress, indirectionIndex, node1, certain) and + storeStepTarget(fieldAddress, indirectionIndex, node2, c) ) or // models-as-data summarized flow diff --git a/cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/AliasedSSA.qll b/cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/AliasedSSA.qll index 2ace50221313..59ee08973e1a 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/AliasedSSA.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/AliasedSSA.qll @@ -295,6 +295,11 @@ abstract class MemoryLocation0 extends TMemoryLocation { */ abstract class VirtualVariable extends MemoryLocation0 { } +pragma[nomagic] +private VirtualVariable getAllocationMemoryLocation(Allocation alloc) { + result.getAnAllocation() = alloc +} + abstract class AllocationMemoryLocation extends MemoryLocation0 { Allocation var; boolean isMayAccess; @@ -313,7 +318,7 @@ abstract class AllocationMemoryLocation extends MemoryLocation0 { result = getGroupedMemoryLocation(var, false, false).getVirtualVariable() or not exists(getGroupedMemoryLocation(var, false, false)) and - result.(AllocationMemoryLocation).getAnAllocation() = var + result = getAllocationMemoryLocation(var) ) } diff --git a/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedInitialization.qll b/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedInitialization.qll index 10c033131225..c24cb98d2bd9 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedInitialization.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedInitialization.qll @@ -618,6 +618,11 @@ class TranslatedExplicitFieldInitialization extends TranslatedNonDefaultFieldIni override int getPosition() { result = position } } +pragma[nomagic] +private Instruction getCallInstruction(TranslatedDefaultFieldInitialization tdfi) { + result = tdfi.getInstruction(CallTag()) +} + /** * The IR translation of the initialization of a field from an element of an initializer * list where default initialization is used. @@ -642,7 +647,7 @@ class TranslatedDefaultFieldInitialization extends TranslatedFieldInitialization override Instruction getInstructionSuccessorInternal(InstructionTag tag, EdgeKind kind) { tag = CallTargetTag() and - result = this.getInstruction(CallTag()) + result = getCallInstruction(this) or tag = CallTag() and result = this.getSideEffects().getFirstInstruction(kind) diff --git a/cpp/ql/src/qlpack.yml b/cpp/ql/src/qlpack.yml index ca971b04aaad..034523449c24 100644 --- a/cpp/ql/src/qlpack.yml +++ b/cpp/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/cpp-queries -version: 1.8.3 +version: 1.8.4-dev groups: - cpp - queries diff --git a/cpp/ql/test/library-tests/dataflow/external-models/bdlbb.cpp b/cpp/ql/test/library-tests/dataflow/external-models/bdlbb.cpp new file mode 100644 index 000000000000..c8ec9dfd031e --- /dev/null +++ b/cpp/ql/test/library-tests/dataflow/external-models/bdlbb.cpp @@ -0,0 +1,98 @@ + +// --- stub library headers --- + +namespace bsl { + typedef unsigned long size_t; + template class allocator {}; + template struct char_traits {}; + template, class Allocator = allocator > + class basic_string { + public: + basic_string(const charT* s, const Allocator& a = Allocator()); + const charT* data() const; + size_t size() const; + }; + typedef basic_string string; + template class shared_ptr { + public: + T *get() const; + }; +} + +namespace BloombergLP { +namespace bdlbb { + class BlobBuffer { + public: + char *data() const; + bsl::shared_ptr &buffer(); + const bsl::shared_ptr &buffer() const; + }; + + class Blob { + public: + const BlobBuffer &buffer(int index) const; + }; + + struct BlobUtil { + static void copy(char *dstBuffer, const Blob &srcBlob, int position, int length); + static void copy(Blob *dstBlob, int dstOffset, const char *srcBuffer, int length); + static void copy(Blob *dstBlob, int dstOffset, const Blob &srcBlob, int srcOffset, + int length); + static char *getContiguousRangeOrCopy(char *dstBuffer, const Blob &srcBlob, int position, + int length, int alignment); + }; +} +} + +// --- test code --- + +char *source(); +void sink(char); + +// A blob populated from a tainted buffer taints the bytes read back out of it. +void test_BlobUtil_copy() { + bsl::string s(source()); + BloombergLP::bdlbb::Blob blob; + BloombergLP::bdlbb::BlobUtil::copy(&blob, 0, s.data(), s.size()); + char dst[16]; + BloombergLP::bdlbb::BlobUtil::copy(dst, blob, 0, 16); + sink(*dst); // $ ir +} + +void test_accessor_chain() { + bsl::string s(source()); + BloombergLP::bdlbb::Blob blob; + BloombergLP::bdlbb::BlobUtil::copy(&blob, 0, s.data(), s.size()); + const char *p = blob.buffer(0).data(); + sink(*p); // $ ir +} + +// The get() step comes from the built-in smart pointer model, not from bdlbb.model.yml. +void test_accessor_chain_shared_ptr() { + bsl::string s(source()); + BloombergLP::bdlbb::Blob blob; + BloombergLP::bdlbb::BlobUtil::copy(&blob, 0, s.data(), s.size()); + const char *p = blob.buffer(0).buffer().get(); + sink(*p); // $ ir +} + +void test_getContiguousRangeOrCopy() { + bsl::string s(source()); + BloombergLP::bdlbb::Blob blob; + BloombergLP::bdlbb::BlobUtil::copy(&blob, 0, s.data(), s.size()); + char dst[16]; + char *r = BloombergLP::bdlbb::BlobUtil::getContiguousRangeOrCopy(dst, blob, 0, 16, 1); + sink(*r); // $ ir +} + +// A blob copied into another blob carries the taint across. +void test_BlobUtil_copy_blob_to_blob() { + bsl::string s(source()); + BloombergLP::bdlbb::Blob src; + BloombergLP::bdlbb::BlobUtil::copy(&src, 0, s.data(), s.size()); + BloombergLP::bdlbb::Blob dst; + BloombergLP::bdlbb::BlobUtil::copy(&dst, 0, src, 0, 16); + char out[16]; + BloombergLP::bdlbb::BlobUtil::copy(out, dst, 0, 16); + sink(*out); // $ ir +} diff --git a/cpp/ql/test/library-tests/dataflow/external-models/flow.expected b/cpp/ql/test/library-tests/dataflow/external-models/flow.expected index 74f24322340f..65817b549a90 100644 --- a/cpp/ql/test/library-tests/dataflow/external-models/flow.expected +++ b/cpp/ql/test/library-tests/dataflow/external-models/flow.expected @@ -95,7 +95,14 @@ models | 94 | Summary: Azure::Core::IO; BodyStream; true; ReadToCount; ; ; Argument[-1]; Argument[*0]; taint; manual | | 95 | Summary: Azure::Core::IO; BodyStream; true; ReadToEnd; ; ; Argument[-1]; ReturnValue.Element; taint; manual | | 96 | Summary: Azure; Nullable; true; Value; ; ; Argument[-1]; ReturnValue[*]; taint; manual | -| 97 | Summary: boost::asio; ; false; buffer; ; ; Argument[*0]; ReturnValue; taint; manual | +| 97 | Summary: BloombergLP::bdlbb; Blob; true; buffer; ; ; Argument[-1]; ReturnValue[*]; taint; manual | +| 98 | Summary: BloombergLP::bdlbb; BlobBuffer; true; buffer; ; ; Argument[-1]; ReturnValue[*]; taint; manual | +| 99 | Summary: BloombergLP::bdlbb; BlobBuffer; true; data; ; ; Argument[-1]; ReturnValue[*]; taint; manual | +| 100 | Summary: BloombergLP::bdlbb; BlobUtil; true; copy; (Blob *,int,const Blob &,int,int); ; Argument[*2]; Argument[*0]; taint; manual | +| 101 | Summary: BloombergLP::bdlbb; BlobUtil; true; copy; (Blob *,int,const char *,int); ; Argument[*2]; Argument[*0]; taint; manual | +| 102 | Summary: BloombergLP::bdlbb; BlobUtil; true; copy; (char *,const Blob &,int,int); ; Argument[*1]; Argument[*0]; taint; manual | +| 103 | Summary: BloombergLP::bdlbb; BlobUtil; true; getContiguousRangeOrCopy; ; ; Argument[*1]; ReturnValue[*]; taint; manual | +| 104 | Summary: boost::asio; ; false; buffer; ; ; Argument[*0]; ReturnValue; taint; manual | edges | asio_streams.cpp:87:34:87:44 | read_until output argument | asio_streams.cpp:91:7:91:17 | recv_buffer | provenance | Src:MaD:56 | | asio_streams.cpp:87:34:87:44 | read_until output argument | asio_streams.cpp:93:29:93:39 | recv_buffer | provenance | Src:MaD:56 Sink:MaD:4 | @@ -104,7 +111,7 @@ edges | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | | | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:101:7:101:17 | send_buffer | provenance | | | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:103:29:103:39 | send_buffer | provenance | Sink:MaD:4 | -| asio_streams.cpp:100:64:100:71 | *send_str | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | MaD:97 | +| asio_streams.cpp:100:64:100:71 | *send_str | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | MaD:104 | | azure.cpp:253:48:253:60 | *call to GetBodyStream | azure.cpp:257:5:257:8 | *resp | provenance | | | azure.cpp:253:48:253:60 | *call to GetBodyStream | azure.cpp:262:5:262:8 | *resp | provenance | | | azure.cpp:253:48:253:60 | *call to GetBodyStream | azure.cpp:266:38:266:41 | *resp | provenance | | @@ -144,6 +151,38 @@ edges | azure.cpp:294:38:294:53 | call to operator[] | azure.cpp:295:10:295:20 | contentType | provenance | | | azure.cpp:294:38:294:53 | call to operator[] | azure.cpp:295:10:295:20 | contentType | provenance | | | azure.cpp:295:10:295:20 | contentType | azure.cpp:295:10:295:20 | contentType | provenance | | +| bdlbb.cpp:54:16:54:23 | call to source | bdlbb.cpp:56:49:56:52 | *call to data | provenance | TaintFunction | +| bdlbb.cpp:56:37:56:41 | copy output argument | bdlbb.cpp:58:42:58:45 | *blob | provenance | | +| bdlbb.cpp:56:49:56:52 | *call to data | bdlbb.cpp:56:37:56:41 | copy output argument | provenance | MaD:101 | +| bdlbb.cpp:58:37:58:39 | copy output argument | bdlbb.cpp:59:7:59:10 | * ... | provenance | | +| bdlbb.cpp:58:42:58:45 | *blob | bdlbb.cpp:58:37:58:39 | copy output argument | provenance | MaD:102 | +| bdlbb.cpp:63:16:63:23 | call to source | bdlbb.cpp:65:49:65:52 | *call to data | provenance | TaintFunction | +| bdlbb.cpp:65:37:65:41 | copy output argument | bdlbb.cpp:66:18:66:21 | *blob | provenance | | +| bdlbb.cpp:65:49:65:52 | *call to data | bdlbb.cpp:65:37:65:41 | copy output argument | provenance | MaD:101 | +| bdlbb.cpp:66:18:66:21 | *blob | bdlbb.cpp:66:29:66:32 | *call to buffer | provenance | MaD:97 | +| bdlbb.cpp:66:18:66:38 | *call to data | bdlbb.cpp:66:18:66:38 | *call to data | provenance | | +| bdlbb.cpp:66:18:66:38 | *call to data | bdlbb.cpp:67:7:67:8 | * ... | provenance | | +| bdlbb.cpp:66:29:66:32 | *call to buffer | bdlbb.cpp:66:18:66:38 | *call to data | provenance | MaD:99 | +| bdlbb.cpp:72:16:72:23 | call to source | bdlbb.cpp:74:49:74:52 | *call to data | provenance | TaintFunction | +| bdlbb.cpp:74:37:74:41 | copy output argument | bdlbb.cpp:75:18:75:21 | *blob | provenance | | +| bdlbb.cpp:74:49:74:52 | *call to data | bdlbb.cpp:74:37:74:41 | copy output argument | provenance | MaD:101 | +| bdlbb.cpp:75:18:75:21 | *blob | bdlbb.cpp:75:29:75:32 | *call to buffer | provenance | MaD:97 | +| bdlbb.cpp:75:18:75:46 | call to get | bdlbb.cpp:76:7:76:8 | * ... | provenance | | +| bdlbb.cpp:75:29:75:32 | *call to buffer | bdlbb.cpp:75:39:75:41 | *call to buffer | provenance | MaD:98 | +| bdlbb.cpp:75:39:75:41 | *call to buffer | bdlbb.cpp:75:18:75:46 | call to get | provenance | DataFlowFunction | +| bdlbb.cpp:80:16:80:23 | call to source | bdlbb.cpp:82:49:82:52 | *call to data | provenance | TaintFunction | +| bdlbb.cpp:82:37:82:41 | copy output argument | bdlbb.cpp:84:72:84:75 | *blob | provenance | | +| bdlbb.cpp:82:49:82:52 | *call to data | bdlbb.cpp:82:37:82:41 | copy output argument | provenance | MaD:101 | +| bdlbb.cpp:84:12:84:65 | *call to getContiguousRangeOrCopy | bdlbb.cpp:84:12:84:65 | *call to getContiguousRangeOrCopy | provenance | | +| bdlbb.cpp:84:12:84:65 | *call to getContiguousRangeOrCopy | bdlbb.cpp:85:7:85:8 | * ... | provenance | | +| bdlbb.cpp:84:72:84:75 | *blob | bdlbb.cpp:84:12:84:65 | *call to getContiguousRangeOrCopy | provenance | MaD:103 | +| bdlbb.cpp:90:16:90:23 | call to source | bdlbb.cpp:92:48:92:51 | *call to data | provenance | TaintFunction | +| bdlbb.cpp:92:37:92:40 | copy output argument | bdlbb.cpp:94:46:94:48 | *src | provenance | | +| bdlbb.cpp:92:48:92:51 | *call to data | bdlbb.cpp:92:37:92:40 | copy output argument | provenance | MaD:101 | +| bdlbb.cpp:94:37:94:40 | copy output argument | bdlbb.cpp:96:42:96:44 | *dst | provenance | | +| bdlbb.cpp:94:46:94:48 | *src | bdlbb.cpp:94:37:94:40 | copy output argument | provenance | MaD:100 | +| bdlbb.cpp:96:37:96:39 | copy output argument | bdlbb.cpp:97:7:97:10 | * ... | provenance | | +| bdlbb.cpp:96:42:96:44 | *dst | bdlbb.cpp:96:37:96:39 | copy output argument | provenance | MaD:102 | | test.cpp:7:47:7:52 | value2 | test.cpp:7:64:7:69 | value2 | provenance | | | test.cpp:7:64:7:69 | value2 | test.cpp:7:5:7:30 | *ymlStepGenerated_with_body | provenance | | | test.cpp:10:10:10:18 | call to ymlSource | test.cpp:10:10:10:18 | call to ymlSource | provenance | Src:MaD:48 | @@ -541,6 +580,43 @@ nodes | azure.cpp:295:10:295:20 | contentType | semmle.label | contentType | | azure.cpp:295:10:295:20 | contentType | semmle.label | contentType | | azure.cpp:295:10:295:20 | contentType | semmle.label | contentType | +| bdlbb.cpp:54:16:54:23 | call to source | semmle.label | call to source | +| bdlbb.cpp:56:37:56:41 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:56:49:56:52 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:58:37:58:39 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:58:42:58:45 | *blob | semmle.label | *blob | +| bdlbb.cpp:59:7:59:10 | * ... | semmle.label | * ... | +| bdlbb.cpp:63:16:63:23 | call to source | semmle.label | call to source | +| bdlbb.cpp:65:37:65:41 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:65:49:65:52 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:66:18:66:21 | *blob | semmle.label | *blob | +| bdlbb.cpp:66:18:66:38 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:66:18:66:38 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:66:29:66:32 | *call to buffer | semmle.label | *call to buffer | +| bdlbb.cpp:67:7:67:8 | * ... | semmle.label | * ... | +| bdlbb.cpp:72:16:72:23 | call to source | semmle.label | call to source | +| bdlbb.cpp:74:37:74:41 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:74:49:74:52 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:75:18:75:21 | *blob | semmle.label | *blob | +| bdlbb.cpp:75:18:75:46 | call to get | semmle.label | call to get | +| bdlbb.cpp:75:29:75:32 | *call to buffer | semmle.label | *call to buffer | +| bdlbb.cpp:75:39:75:41 | *call to buffer | semmle.label | *call to buffer | +| bdlbb.cpp:76:7:76:8 | * ... | semmle.label | * ... | +| bdlbb.cpp:80:16:80:23 | call to source | semmle.label | call to source | +| bdlbb.cpp:82:37:82:41 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:82:49:82:52 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:84:12:84:65 | *call to getContiguousRangeOrCopy | semmle.label | *call to getContiguousRangeOrCopy | +| bdlbb.cpp:84:12:84:65 | *call to getContiguousRangeOrCopy | semmle.label | *call to getContiguousRangeOrCopy | +| bdlbb.cpp:84:72:84:75 | *blob | semmle.label | *blob | +| bdlbb.cpp:85:7:85:8 | * ... | semmle.label | * ... | +| bdlbb.cpp:90:16:90:23 | call to source | semmle.label | call to source | +| bdlbb.cpp:92:37:92:40 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:92:48:92:51 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:94:37:94:40 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:94:46:94:48 | *src | semmle.label | *src | +| bdlbb.cpp:96:37:96:39 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:96:42:96:44 | *dst | semmle.label | *dst | +| bdlbb.cpp:97:7:97:10 | * ... | semmle.label | * ... | | test.cpp:7:5:7:30 | *ymlStepGenerated_with_body | semmle.label | *ymlStepGenerated_with_body | | test.cpp:7:47:7:52 | value2 | semmle.label | value2 | | test.cpp:7:64:7:69 | value2 | semmle.label | value2 | diff --git a/cpp/ql/test/library-tests/dataflow/external-models/steps.expected b/cpp/ql/test/library-tests/dataflow/external-models/steps.expected index 0fe13460cfbf..42d2c0183c34 100644 --- a/cpp/ql/test/library-tests/dataflow/external-models/steps.expected +++ b/cpp/ql/test/library-tests/dataflow/external-models/steps.expected @@ -4,6 +4,20 @@ | azure.cpp:262:5:262:8 | *resp | azure.cpp:262:23:262:28 | ReadToCount output argument | | azure.cpp:287:79:287:98 | call to string | azure.cpp:287:62:287:99 | call to Url | | azure.cpp:289:24:289:56 | call to GetHeader | azure.cpp:289:63:289:65 | call to Value | +| bdlbb.cpp:56:49:56:52 | *call to data | bdlbb.cpp:56:37:56:41 | copy output argument | +| bdlbb.cpp:58:42:58:45 | *blob | bdlbb.cpp:58:37:58:39 | copy output argument | +| bdlbb.cpp:65:49:65:52 | *call to data | bdlbb.cpp:65:37:65:41 | copy output argument | +| bdlbb.cpp:66:18:66:21 | *blob | bdlbb.cpp:66:29:66:32 | *call to buffer | +| bdlbb.cpp:66:29:66:32 | *call to buffer | bdlbb.cpp:66:18:66:38 | *call to data | +| bdlbb.cpp:74:49:74:52 | *call to data | bdlbb.cpp:74:37:74:41 | copy output argument | +| bdlbb.cpp:75:18:75:21 | *blob | bdlbb.cpp:75:29:75:32 | *call to buffer | +| bdlbb.cpp:75:29:75:32 | *call to buffer | bdlbb.cpp:75:39:75:41 | *call to buffer | +| bdlbb.cpp:82:49:82:52 | *call to data | bdlbb.cpp:82:37:82:41 | copy output argument | +| bdlbb.cpp:84:72:84:75 | *blob | bdlbb.cpp:84:12:84:65 | *call to getContiguousRangeOrCopy | +| bdlbb.cpp:84:72:84:75 | *blob | bdlbb.cpp:84:67:84:69 | getContiguousRangeOrCopy output argument | +| bdlbb.cpp:92:48:92:51 | *call to data | bdlbb.cpp:92:37:92:40 | copy output argument | +| bdlbb.cpp:94:46:94:48 | *src | bdlbb.cpp:94:37:94:40 | copy output argument | +| bdlbb.cpp:96:42:96:44 | *dst | bdlbb.cpp:96:37:96:39 | copy output argument | | test.cpp:17:24:17:24 | x | test.cpp:17:10:17:22 | call to ymlStepManual | | test.cpp:21:27:21:27 | x | test.cpp:21:10:21:25 | call to ymlStepGenerated | | test.cpp:25:35:25:35 | x | test.cpp:25:11:25:33 | call to ymlStepManual_with_body | diff --git a/cpp/ql/test/library-tests/dataflow/external-models/validatemodels.expected b/cpp/ql/test/library-tests/dataflow/external-models/validatemodels.expected index 15ae50bddc26..1fbe5da66459 100644 --- a/cpp/ql/test/library-tests/dataflow/external-models/validatemodels.expected +++ b/cpp/ql/test/library-tests/dataflow/external-models/validatemodels.expected @@ -370,6 +370,8 @@ | Dubious signature "(BN_MONT_CTX *,const BIGNUM *,int,const unsigned char *,size_t,uint32_t,uint32_t)" in summary model. | | Dubious signature "(BN_RECP_CTX *,const BIGNUM *,BN_CTX *)" in summary model. | | Dubious signature "(BUF_MEM *,size_t)" in summary model. | +| Dubious signature "(Blob *,int,const Blob &,int,int)" in summary model. | +| Dubious signature "(Blob *,int,const char *,int)" in summary model. | | Dubious signature "(BrotliBitReader *const,uint64_t,uint64_t *)" in summary model. | | Dubious signature "(BrotliDecoderState *,BrotliDecoderStateInternal *,BrotliSharedDictionaryType,size_t,const uint8_t[])" in summary model. | | Dubious signature "(BrotliDecoderState *,BrotliDecoderStateInternal *,brotli_decoder_metadata_start_func,brotli_decoder_metadata_chunk_func,void *)" in summary model. | @@ -2948,6 +2950,7 @@ | Dubious signature "(char *,char *__restrict__,int,FILE *,FILE *__restrict__)" in summary model. | | Dubious signature "(char *,char *__restrict__,size_t,const char *,const char *__restrict__,const tm *,const tm *__restrict__,locale_t)" in summary model. | | Dubious signature "(char *,char,char **)" in summary model. | +| Dubious signature "(char *,const Blob &,int,int)" in summary model. | | Dubious signature "(char *,const char *)" in summary model. | | Dubious signature "(char *,const char **,const char **,const char **,const char **,const char **)" in summary model. | | Dubious signature "(char *,const char *,char **)" in summary model. | diff --git a/cpp/ql/test/library-tests/dataflow/taint-tests/test_mad-signatures.expected b/cpp/ql/test/library-tests/dataflow/taint-tests/test_mad-signatures.expected index d494c09e71d5..2da7e83cca37 100644 --- a/cpp/ql/test/library-tests/dataflow/taint-tests/test_mad-signatures.expected +++ b/cpp/ql/test/library-tests/dataflow/taint-tests/test_mad-signatures.expected @@ -1962,6 +1962,15 @@ getSignatureParameterName | (BUF_MEM *,size_t) | | BUF_MEM_grow | 1 | size_t | | (BUF_MEM *,size_t) | | BUF_MEM_grow_clean | 0 | BUF_MEM * | | (BUF_MEM *,size_t) | | BUF_MEM_grow_clean | 1 | size_t | +| (Blob *,int,const Blob &,int,int) | BlobUtil | copy | 0 | Blob * | +| (Blob *,int,const Blob &,int,int) | BlobUtil | copy | 1 | int | +| (Blob *,int,const Blob &,int,int) | BlobUtil | copy | 2 | const Blob & | +| (Blob *,int,const Blob &,int,int) | BlobUtil | copy | 3 | int | +| (Blob *,int,const Blob &,int,int) | BlobUtil | copy | 4 | int | +| (Blob *,int,const char *,int) | BlobUtil | copy | 0 | Blob * | +| (Blob *,int,const char *,int) | BlobUtil | copy | 1 | int | +| (Blob *,int,const char *,int) | BlobUtil | copy | 2 | const char * | +| (Blob *,int,const char *,int) | BlobUtil | copy | 3 | int | | (BrotliBitReader *const,uint64_t,uint64_t *) | | BrotliSafeReadBits32Slow | 0 | BrotliBitReader *const | | (BrotliBitReader *const,uint64_t,uint64_t *) | | BrotliSafeReadBits32Slow | 1 | uint64_t | | (BrotliBitReader *const,uint64_t,uint64_t *) | | BrotliSafeReadBits32Slow | 2 | uint64_t * | @@ -13127,6 +13136,10 @@ getSignatureParameterName | (char *,char,char **) | | __old_strtok_r_1c | 0 | char * | | (char *,char,char **) | | __old_strtok_r_1c | 1 | char | | (char *,char,char **) | | __old_strtok_r_1c | 2 | char ** | +| (char *,const Blob &,int,int) | BlobUtil | copy | 0 | char * | +| (char *,const Blob &,int,int) | BlobUtil | copy | 1 | const Blob & | +| (char *,const Blob &,int,int) | BlobUtil | copy | 2 | int | +| (char *,const Blob &,int,int) | BlobUtil | copy | 3 | int | | (char *,const char *) | | xstrdup | 0 | char * | | (char *,const char *) | | xstrdup | 1 | const char * | | (char *,const char **,const char **,const char **,const char **,const char **) | | _nl_explode_name | 0 | char * | diff --git a/csharp/ql/campaigns/Solorigate/lib/qlpack.yml b/csharp/ql/campaigns/Solorigate/lib/qlpack.yml index 990c6ad4dbc8..07c306a65a9d 100644 --- a/csharp/ql/campaigns/Solorigate/lib/qlpack.yml +++ b/csharp/ql/campaigns/Solorigate/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-solorigate-all -version: 1.7.74 +version: 1.7.75-dev groups: - csharp - solorigate diff --git a/csharp/ql/campaigns/Solorigate/src/qlpack.yml b/csharp/ql/campaigns/Solorigate/src/qlpack.yml index 199504aa7dd3..94bf9c0f7db3 100644 --- a/csharp/ql/campaigns/Solorigate/src/qlpack.yml +++ b/csharp/ql/campaigns/Solorigate/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-solorigate-queries -version: 1.7.74 +version: 1.7.75-dev groups: - csharp - solorigate diff --git a/csharp/ql/lib/qlpack.yml b/csharp/ql/lib/qlpack.yml index 15fadfad8a0f..18ee2c149098 100644 --- a/csharp/ql/lib/qlpack.yml +++ b/csharp/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-all -version: 7.3.0 +version: 7.3.1-dev groups: csharp dbscheme: semmlecode.csharp.dbscheme extractor: csharp diff --git a/csharp/ql/src/qlpack.yml b/csharp/ql/src/qlpack.yml index a2f6f0aac243..c90064a3b913 100644 --- a/csharp/ql/src/qlpack.yml +++ b/csharp/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-queries -version: 1.9.3 +version: 1.9.4-dev groups: - csharp - queries diff --git a/docs/codeql/reusables/supported-versions-compilers.rst b/docs/codeql/reusables/supported-versions-compilers.rst index 8651fa91a269..e341918e5931 100644 --- a/docs/codeql/reusables/supported-versions-compilers.rst +++ b/docs/codeql/reusables/supported-versions-compilers.rst @@ -4,9 +4,9 @@ :stub-columns: 1 Language,Variants,Compilers,Extensions - C/C++,"C89, C99, C11, C17, C23, C++98, C++03, C++11, C++14, C++17, C++20, C++23 [1]_ [2]_ [3]_","Clang (including clang-cl and armclang) extensions (up to Clang 21), + C/C++,"C89, C99, C11, C17, C23, C++98, C++03, C++11, C++14, C++17, C++20, C++23 [1]_ [2]_ [3]_","Clang (including clang-cl and armclang) extensions (up to Clang 22), - GNU extensions (up to GCC 15), + GNU extensions (up to GCC 16), Microsoft extensions (up to VS 2022), diff --git a/go/ql/consistency-queries/qlpack.yml b/go/ql/consistency-queries/qlpack.yml index 353e4dd1cdcd..c9c318de2fa2 100644 --- a/go/ql/consistency-queries/qlpack.yml +++ b/go/ql/consistency-queries/qlpack.yml @@ -1,5 +1,5 @@ name: codeql-go-consistency-queries -version: 1.0.57 +version: 1.0.58-dev groups: - go - queries diff --git a/go/ql/lib/qlpack.yml b/go/ql/lib/qlpack.yml index fb42dcb703ae..4dc828abeffe 100644 --- a/go/ql/lib/qlpack.yml +++ b/go/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/go-all -version: 7.3.1 +version: 7.3.2-dev groups: go dbscheme: go.dbscheme extractor: go diff --git a/go/ql/src/qlpack.yml b/go/ql/src/qlpack.yml index bc9f243309af..fdfdd1f67432 100644 --- a/go/ql/src/qlpack.yml +++ b/go/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/go-queries -version: 1.6.10 +version: 1.6.11-dev groups: - go - queries diff --git a/java/kotlin-extractor/src/main/java/com/semmle/util/files/FileUtil.java b/java/kotlin-extractor/src/main/java/com/semmle/util/files/FileUtil.java index 19bdc786c5e2..16df41e5341e 100644 --- a/java/kotlin-extractor/src/main/java/com/semmle/util/files/FileUtil.java +++ b/java/kotlin-extractor/src/main/java/com/semmle/util/files/FileUtil.java @@ -1244,8 +1244,8 @@ public static File tryMakeCanonical (File f) try { // getCanonicalFile does not canonicalize subst drives on Windows, so do this separately. This // is a no-op on non-Windows platforms. - return SubstResolver.resolve(f.getCanonicalFile()); } - catch (IOException ignored) { + return SubstResolver.resolve(f.getCanonicalFile()); + } catch (IOException ignored) { Exceptions.ignore(ignored, "Can't log error: Could be too verbose."); return new File(simplifyPath(f)); } diff --git a/java/ql/integration-tests/java/gradle-sample-without-wrapper-or-gradle-buildless/test.py b/java/ql/integration-tests/java/gradle-sample-without-wrapper-or-gradle-buildless/test.py index 3aaee01f4055..a312878e107f 100644 --- a/java/ql/integration-tests/java/gradle-sample-without-wrapper-or-gradle-buildless/test.py +++ b/java/ql/integration-tests/java/gradle-sample-without-wrapper-or-gradle-buildless/test.py @@ -6,7 +6,9 @@ # The version of gradle used doesn't work on java 17 def test(codeql, use_java_11, java, environment, check_diagnostics): check_diagnostics.redact += ["attributes.java_vendor"] - check_diagnostics.replacements = [("11\\.[0-9]+\\.[0-9]+", "11")] + # the JDK build provided by the CI runner image may report any number of version components + # (e.g. `11.0.32` or `11.0.32.1`), so keep only the feature version + check_diagnostics.replacements = [(r'"11(\.[0-9]+)+"', '"11"')] gradle_override_dir = pathlib.Path(tempfile.mkdtemp()) if runs_on.windows: (gradle_override_dir / "gradle.bat").write_text("@echo off\nexit /b 2\n") diff --git a/java/ql/lib/qlpack.yml b/java/ql/lib/qlpack.yml index d97a58c5c809..d6aefad45233 100644 --- a/java/ql/lib/qlpack.yml +++ b/java/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/java-all -version: 9.3.0 +version: 9.3.1-dev groups: java dbscheme: config/semmlecode.dbscheme extractor: java diff --git a/java/ql/src/qlpack.yml b/java/ql/src/qlpack.yml index 9c3cffa047ca..1bb0cdc16c09 100644 --- a/java/ql/src/qlpack.yml +++ b/java/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/java-queries -version: 1.11.10 +version: 1.11.11-dev groups: - java - queries diff --git a/javascript/ql/lib/qlpack.yml b/javascript/ql/lib/qlpack.yml index 97f274ff58ae..93c4370aab6d 100644 --- a/javascript/ql/lib/qlpack.yml +++ b/javascript/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/javascript-all -version: 2.10.1 +version: 2.10.2-dev groups: javascript dbscheme: semmlecode.javascript.dbscheme extractor: javascript diff --git a/javascript/ql/src/qlpack.yml b/javascript/ql/src/qlpack.yml index 83e2caff741c..456c7f4ca7b5 100644 --- a/javascript/ql/src/qlpack.yml +++ b/javascript/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/javascript-queries -version: 2.4.5 +version: 2.4.6-dev groups: - javascript - queries diff --git a/misc/suite-helpers/qlpack.yml b/misc/suite-helpers/qlpack.yml index ba51f45be36f..1c27d6177555 100644 --- a/misc/suite-helpers/qlpack.yml +++ b/misc/suite-helpers/qlpack.yml @@ -1,4 +1,4 @@ name: codeql/suite-helpers -version: 1.0.57 +version: 1.0.58-dev groups: shared warnOnImplicitThis: true diff --git a/python/extractor/cli-integration-test/parser-telemetry/repo_dir/old_parser.py b/python/extractor/cli-integration-test/parser-telemetry/repo_dir/old_parser.py new file mode 100644 index 000000000000..7d4290a117a4 --- /dev/null +++ b/python/extractor/cli-integration-test/parser-telemetry/repo_dir/old_parser.py @@ -0,0 +1 @@ +x = 1 diff --git a/python/extractor/cli-integration-test/parser-telemetry/repo_dir/tree_sitter_parser.py b/python/extractor/cli-integration-test/parser-telemetry/repo_dir/tree_sitter_parser.py new file mode 100644 index 000000000000..08a86e1f4c6d --- /dev/null +++ b/python/extractor/cli-integration-test/parser-telemetry/repo_dir/tree_sitter_parser.py @@ -0,0 +1,3 @@ +match 1: + case 1: + pass diff --git a/python/extractor/cli-integration-test/parser-telemetry/test.sh b/python/extractor/cli-integration-test/parser-telemetry/test.sh new file mode 100755 index 000000000000..86219c86bf12 --- /dev/null +++ b/python/extractor/cli-integration-test/parser-telemetry/test.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ + +set -x + +CODEQL=${CODEQL:-codeql} + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$SCRIPTDIR" + +rm -rf db + +$CODEQL database create db --language python --source-root repo_dir/ +python3 test_parser_telemetry.py db + +rm -rf db diff --git a/python/extractor/cli-integration-test/parser-telemetry/test_parser_telemetry.py b/python/extractor/cli-integration-test/parser-telemetry/test_parser_telemetry.py new file mode 100644 index 000000000000..c848f63eacd5 --- /dev/null +++ b/python/extractor/cli-integration-test/parser-telemetry/test_parser_telemetry.py @@ -0,0 +1,25 @@ +import glob +import json +import os +import sys + + +database = sys.argv[1] +diagnostics = [] +diagnostic_dir = os.path.join(database, "diagnostic", "extractors", "python") +for path in glob.glob(os.path.join(diagnostic_dir, "*.jsonl")): + with open(path) as diagnostic_file: + diagnostics.extend(json.loads(line) for line in diagnostic_file) +parser_statistics = [ + diagnostic + for diagnostic in diagnostics + if diagnostic["source"]["id"] == "py/extractor/parser-statistics" +] +actual = ( + sum(diagnostic["attributes"]["old_parser_file_count"] for diagnostic in parser_statistics), + sum( + diagnostic["attributes"]["tree_sitter_parser_file_count"] + for diagnostic in parser_statistics + ), +) +assert actual == (1, 1), actual diff --git a/python/extractor/cli-integration-test/writing-diagnostics/diagnostics.expected b/python/extractor/cli-integration-test/writing-diagnostics/diagnostics.expected index 12a241ad7b68..4b6fceb48faf 100644 --- a/python/extractor/cli-integration-test/writing-diagnostics/diagnostics.expected +++ b/python/extractor/cli-integration-test/writing-diagnostics/diagnostics.expected @@ -161,3 +161,24 @@ "telemetry": true } } +{ + "attributes": { + "extractor_flags": "default", + "extractor_version": "7.1.10", + "python_analysis_version": "3.12", + "python_runtime_version": "3.12.3" + }, + "markdownMessage": "Internal telemetry for the Python extractor.\n\nNo action needed.", + "severity": "note", + "source": { + "extractorName": "python", + "id": "py/extractor/summary", + "name": "Python extractor telemetry" + }, + "timestamp": "2026-09-01T13:41:33.056818Z", + "visibility": { + "cliSummaryTable": false, + "statusPage": false, + "telemetry": true + } +} diff --git a/python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py b/python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py index 0dce022a0f95..4108b9731943 100644 --- a/python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py +++ b/python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py @@ -1,7 +1,28 @@ import os import sys +import glob +import json sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..", "integration-tests")) import diagnostics_test_utils test_db = "db" -diagnostics_test_utils.check_diagnostics(".", test_db, skip_attributes=True) +diagnostics = [] +diagnostic_dir = os.path.join(test_db, "diagnostic", "extractors", "python") +for path in glob.glob(os.path.join(diagnostic_dir, "*.jsonl")): + with open(path) as diagnostic_file: + diagnostics.extend(json.loads(line) for line in diagnostic_file) +summary = [ + diagnostic + for diagnostic in diagnostics + if diagnostic["source"]["id"] == "py/extractor/summary" +] +assert len(summary) == 1 +assert summary[0]["attributes"]["extractor_flags"] == "default" +diagnostics_test_utils.check_diagnostics( + ".", + test_db, + skip_attributes=True, + replacements={ + r'"py/extractor/parser-statistics"': '"cli/py/extractor/parser-statistics"' + }, +) diff --git a/python/extractor/semmle/cmdline.py b/python/extractor/semmle/cmdline.py index 47007c065fdc..213a1f4affae 100644 --- a/python/extractor/semmle/cmdline.py +++ b/python/extractor/semmle/cmdline.py @@ -1,4 +1,4 @@ -from optparse import OptionParser, OptionGroup, HelpFormatter +from optparse import Option, OptionParser, OptionGroup, HelpFormatter import shlex import sys import os @@ -8,9 +8,21 @@ from semmle.util import VERSION +DEFAULT_AUTOBUILDER_FLAGS = {"R", "c", "v", "verbosity", "z"} + + +class RecordingOption(Option): + def process(self, opt, value, values, parser): + flag = (self._short_opts or self._long_opts)[0].lstrip("-") + if flag not in DEFAULT_AUTOBUILDER_FLAGS: + parser.extractor_flags.add(flag) + return Option.process(self, opt, value, values, parser) + + def make_parser(): '''Parse command_line, returning options, arguments''' - parser = OptionParser(add_help_option=False, version='%s' % VERSION) + parser = OptionParser(option_class=RecordingOption, add_help_option=False, version='%s' % VERSION) + parser.extractor_flags = set() import_options = OptionGroup(parser, "Import following options", description="Note that -a -n -g and -t are included for backwards compatibility. They are ignored") @@ -172,6 +184,7 @@ def parse(command_line): setattr(options, attr, dval) args.extend(extra_args) del options.file + options.extractor_flags = sorted(parser.extractor_flags) if options.help: if options.verbose: for opt in parser._get_all_options(): diff --git a/python/extractor/semmle/extractors/module_printer.py b/python/extractor/semmle/extractors/module_printer.py index d2f4a6cc92bd..aeaa8e63f5eb 100644 --- a/python/extractor/semmle/extractors/module_printer.py +++ b/python/extractor/semmle/extractors/module_printer.py @@ -6,9 +6,9 @@ class ModulePrinter(object): name = "module printer" - def __init__(self, options, trap_folder, src_archive, renamer, logger): + def __init__(self, options, trap_folder, src_archive, renamer, logger, diagnostics_writer): self.logger = logger - self.py_extractor = PythonExtractor(options, trap_folder, src_archive, logger) + self.py_extractor = PythonExtractor(options, trap_folder, src_archive, logger, diagnostics_writer) def process(self, unit): imports = () diff --git a/python/extractor/semmle/extractors/py_extractor.py b/python/extractor/semmle/extractors/py_extractor.py index 8014063b3cb8..3b5a37fb7a96 100644 --- a/python/extractor/semmle/extractors/py_extractor.py +++ b/python/extractor/semmle/extractors/py_extractor.py @@ -16,6 +16,7 @@ def __init__(self, options, trap_folder, src_archive, logger: Logger, diagnostic self.module_extractor = extractor.Extractor.from_options(options, trap_folder, src_archive, logger, diagnostics_writer) self.finder = finder.Finder.from_options_and_env(options, logger) self.importer = imports.importer_from_options(options, self.finder, logger) + self.diagnostics_writer = diagnostics_writer def _get_module_and_imports(self, unit): if not isinstance(unit, util.FileExtractable): @@ -24,7 +25,7 @@ def _get_module_and_imports(self, unit): module = self.finder.from_extractable(unit) if module is None: return None, () - py_module = module.load(self.logger) + py_module = module.load(self.logger, self.diagnostics_writer) if py_module is None: return None, () imports = set(mod.get_extractable() for mod in self.importer.get_imports(module, py_module)) diff --git a/python/extractor/semmle/logging.py b/python/extractor/semmle/logging.py index 64037163e697..652ab810d0fb 100644 --- a/python/extractor/semmle/logging.py +++ b/python/extractor/semmle/logging.py @@ -8,6 +8,9 @@ import multiprocessing import enum import datetime +import platform + +from semmle.util import VERSION, get_analysis_version #Use standard Semmle logging levels @@ -355,6 +358,24 @@ def with_timestamp(self, timestamp): self.timestamp = timestamp return self +def extractor_telemetry_message(extractor_flags): + return (DiagnosticMessage(Source("py/extractor/summary", "Python extractor telemetry"), Severity.NOTE) + .markdown("Internal telemetry for the Python extractor.\n\nNo action needed.") + .attribute("python_analysis_version", get_analysis_version()) + .attribute("python_runtime_version", platform.python_version()) + .attribute("extractor_version", VERSION) + .attribute("extractor_flags", " ".join(extractor_flags) or "default") + .telemetry() + ) + +def parser_statistics_telemetry_message(old_parser_file_count, tree_sitter_parser_file_count): + return (DiagnosticMessage(Source("py/extractor/parser-statistics", "Python parser statistics"), Severity.NOTE) + .markdown("Internal parser telemetry for the Python extractor.\n\nNo action needed.") + .attribute("old_parser_file_count", old_parser_file_count) + .attribute("tree_sitter_parser_file_count", tree_sitter_parser_file_count) + .telemetry() + ) + def get_stack_trace_lines(): """Creates a stack trace for inclusion into the `attributes` part of a diagnostic message. Limits the size of the stack trace to 5000 characters, so as to not make the SARIF file overly big. diff --git a/python/extractor/semmle/python/finder.py b/python/extractor/semmle/python/finder.py index 632ef920d055..46e63f5e578f 100644 --- a/python/extractor/semmle/python/finder.py +++ b/python/extractor/semmle/python/finder.py @@ -65,8 +65,8 @@ def all_sub_modules(self): def get_extractable(self): return FileExtractable(self.path) - def load(self, logger=None): - return PythonSourceModule(self.name, self.path, logger=logger) + def load(self, logger, diagnostics_writer): + return PythonSourceModule(self.name, self.path, logger=logger, diagnostics_writer=diagnostics_writer) def __str__(self): return "Python module at %s" % self.path diff --git a/python/extractor/semmle/python/modules.py b/python/extractor/semmle/python/modules.py index 810c4e060f7b..7192f97cfb11 100644 --- a/python/extractor/semmle/python/modules.py +++ b/python/extractor/semmle/python/modules.py @@ -18,7 +18,7 @@ class PythonSourceModule(object): kind = None - def __init__(self, name, path, logger, bytes_source = None): + def __init__(self, name, path, logger, diagnostics_writer, bytes_source = None): assert isinstance(path, str), path self.name = name # May be None self.path = path @@ -34,6 +34,7 @@ def __init__(self, name, path, logger, bytes_source = None): self._line_types = None self._comments = None self._tokens = None + self.diagnostics_writer = diagnostics_writer self.logger = logger with timers["decode"]: self.encoding, self.bytes_source = semmle.python.parser.tokenizer.encoding_from_source(bytes_source) @@ -113,6 +114,7 @@ def old_py_ast(self): self.logger.debug("Trying old parser on %s", self.path) self._py_ast = semmle.python.parser.parse(self.tokens, self.logger) self.logger.debug("Old parser successful on %s", self.path) + self.diagnostics_writer.record_old_parser() else: self.logger.debug("Found (during old_py_ast) parse tree for %s in cache", self.path) return self._py_ast @@ -147,6 +149,7 @@ def py_ast(self): self.logger.debug("Trying tsg-python on %s", self.path) self._py_ast = semmle.python.parser.tsg_parser.parse(self.path, self.logger) self.logger.debug("tsg-python successful on %s", self.path) + self.diagnostics_writer.record_tree_sitter_parser() else: self.logger.debug("Found (during py_ast) parse tree for %s in cache", self.path) return self._py_ast diff --git a/python/extractor/semmle/python/parser/dump_ast.py b/python/extractor/semmle/python/parser/dump_ast.py index 3a7db5ab0713..97abb502e59a 100644 --- a/python/extractor/semmle/python/parser/dump_ast.py +++ b/python/extractor/semmle/python/parser/dump_ast.py @@ -119,7 +119,8 @@ def reset_error_count(self): self.error_count = 0 def old_parser(inputfile, logger): - mod = PythonSourceModule(None, inputfile, logger) + from semmle.worker import DiagnosticsWriter + mod = PythonSourceModule(None, inputfile, logger, DiagnosticsWriter(0)) logger.close() return mod.old_py_ast diff --git a/python/extractor/semmle/python/passes/flow.py b/python/extractor/semmle/python/passes/flow.py index 6ea5405a8540..a100bbae7c9c 100755 --- a/python/extractor/semmle/python/passes/flow.py +++ b/python/extractor/semmle/python/passes/flow.py @@ -1916,7 +1916,8 @@ def write_ssa_phi(out, phi, arg): import semmle.python.parser.tsg_parser parsed_ast = semmle.python.parser.tsg_parser.parse(inputfile, FakeLogger()) else: - module = modules.PythonSourceModule("__main__", inputfile, FakeLogger()) + from semmle.worker import DiagnosticsWriter + module = modules.PythonSourceModule("__main__", inputfile, FakeLogger(), DiagnosticsWriter(0)) parsed_ast = module.ast FlowPass(options.split, options.prune, options.unroll).extract(parsed_ast, writer) writer.close() diff --git a/python/extractor/semmle/util.py b/python/extractor/semmle/util.py index 977d47c69dca..60d215e5bdf4 100644 --- a/python/extractor/semmle/util.py +++ b/python/extractor/semmle/util.py @@ -10,7 +10,7 @@ #Semantic version of extractor. #Update this if any changes are made -VERSION = "7.1.9" +VERSION = "7.1.10" PY_EXTENSIONS = ".py", ".pyw" diff --git a/python/extractor/semmle/worker.py b/python/extractor/semmle/worker.py index ac8231390742..15b7b556bf88 100644 --- a/python/extractor/semmle/worker.py +++ b/python/extractor/semmle/worker.py @@ -10,7 +10,8 @@ from semmle.extractors import SuperExtractor, ModulePrinter, SkippedBuiltin from semmle.profiling import get_profiler from semmle.path_rename import renamer_from_options_and_env -from semmle.logging import WARN, recursion_error_message, internal_error_message, Logger +from semmle.logging import WARN, recursion_error_message, internal_error_message, extractor_telemetry_message, Logger +from semmle.logging import parser_statistics_telemetry_message from semmle.util import FileExtractable, FolderExtractable class ExtractorFailure(Exception): @@ -239,9 +240,35 @@ def _drain_queue(queue): #Emptied queue as best we can. pass +def _write_extractor_telemetry(diagnostics_writer, logger: Logger, extractor_flags): + try: + diagnostics_writer.write(extractor_telemetry_message(extractor_flags)) + except OSError as ex: + logger.warning("Failed to write extractor telemetry: %s", ex) + +def _write_parser_statistics_telemetry(diagnostics_writer, logger: Logger): + counts = diagnostics_writer.parser_statistics() + if counts == (0, 0): + return + try: + diagnostics_writer.write(parser_statistics_telemetry_message(*counts)) + except OSError as ex: + logger.warning("Failed to write parser statistics telemetry: %s", ex) + class DiagnosticsWriter(object): def __init__(self, proc_id): self.proc_id = proc_id + self.old_parser_file_count = 0 + self.tree_sitter_parser_file_count = 0 + + def record_old_parser(self): + self.old_parser_file_count += 1 + + def record_tree_sitter_parser(self): + self.tree_sitter_parser_file_count += 1 + + def parser_statistics(self): + return self.old_parser_file_count, self.tree_sitter_parser_file_count def write(self, message): dir = os.environ.get("CODEQL_EXTRACTOR_PYTHON_DIAGNOSTIC_DIR") @@ -276,9 +303,11 @@ def _extract_loop(proc_id, queue, trap_dir, archive, options, reply_queue, logge reply_queue.put(("INTERRUPT", None, None)) sys.exit(2) logger.set_process_id(proc_id) + if write_global_data: + _write_extractor_telemetry(diagnostics_writer, logger, options.extractor_flags) try: if options.trace_only: - extractor = ModulePrinter(options, trap_dir, archive, renamer, logger) + extractor = ModulePrinter(options, trap_dir, archive, renamer, logger, diagnostics_writer) else: extractor = SuperExtractor(options, trap_dir, archive, renamer, logger, diagnostics_writer) profiler = get_profiler(options, id, logger) @@ -291,6 +320,7 @@ def _extract_loop(proc_id, queue, trap_dir, archive, options, reply_queue, logge if write_global_data: extractor.write_global_data() extractor.close() + _write_parser_statistics_telemetry(diagnostics_writer, logger) return try: start = time.time() @@ -344,4 +374,5 @@ def _extract_loop(proc_id, queue, trap_dir, archive, options, reply_queue, logge except _Empty: #Cleared queue enough to avoid deadlock. pass + _write_parser_statistics_telemetry(diagnostics_writer, logger) sys.exit(2) diff --git a/python/extractor/tests/test_cmdline.py b/python/extractor/tests/test_cmdline.py new file mode 100644 index 000000000000..76f2fe951b57 --- /dev/null +++ b/python/extractor/tests/test_cmdline.py @@ -0,0 +1,34 @@ +from semmle import cmdline + + +def test_records_flags_without_values(): + options, args = cmdline.parse( + [ + "--verbosity=3", + "-zall", + "-R", + "/src", + "-vv", + "--path", + "/lib", + "-p", + "/other-lib", + "module", + ] + ) + + assert options.extractor_flags == ["p"] + assert args == ["module"] + + +def test_records_flags_from_option_file(tmp_path): + options_file = tmp_path / "extractor-options" + options_file.write_text("--colorize --max-import-depth 2") + + options, _ = cmdline.parse(["-f", str(options_file)]) + + assert options.extractor_flags == [ + "colorize", + "f", + "max-import-depth", + ] diff --git a/python/extractor/tests/test_diagnostics.py b/python/extractor/tests/test_diagnostics.py new file mode 100644 index 000000000000..ddfd22436de9 --- /dev/null +++ b/python/extractor/tests/test_diagnostics.py @@ -0,0 +1,167 @@ +import platform + +from semmle import logging +from semmle import util +from semmle import worker +from semmle.python.modules import PythonSourceModule + + +def test_extractor_telemetry_message(mocker): + mocker.patch("semmle.logging.get_analysis_version", return_value="3.13") + + message = logging.extractor_telemetry_message(["colorize", "p"]).to_dict() + message.pop("timestamp") + + assert message == { + "source": { + "id": "py/extractor/summary", + "name": "Python extractor telemetry", + "extractorName": "python", + }, + "severity": "note", + "markdownMessage": "Internal telemetry for the Python extractor.\n\nNo action needed.", + "visibility": { + "statusPage": False, + "cliSummaryTable": False, + "telemetry": True, + }, + "attributes": { + "python_analysis_version": "3.13", + "python_runtime_version": platform.python_version(), + "extractor_version": util.VERSION, + "extractor_flags": "colorize p", + }, + } + + +def test_parser_statistics_telemetry_message(): + message = logging.parser_statistics_telemetry_message( + old_parser_file_count=12, tree_sitter_parser_file_count=3 + ).to_dict() + message.pop("timestamp") + + assert message == { + "source": { + "id": "py/extractor/parser-statistics", + "name": "Python parser statistics", + "extractorName": "python", + }, + "severity": "note", + "markdownMessage": "Internal parser telemetry for the Python extractor.\n\nNo action needed.", + "visibility": { + "statusPage": False, + "cliSummaryTable": False, + "telemetry": True, + }, + "attributes": { + "old_parser_file_count": 12, + "tree_sitter_parser_file_count": 3, + }, + } + + +def test_extractor_telemetry_message_includes_empty_flags(): + message = logging.extractor_telemetry_message([]).to_dict() + + assert message["attributes"]["extractor_flags"] == "default" + + +def test_write_extractor_telemetry(mocker): + diagnostics_writer = mocker.Mock() + logger = mocker.Mock() + + worker._write_extractor_telemetry(diagnostics_writer, logger, ["quiet"]) + + diagnostics_writer.write.assert_called_once() + assert diagnostics_writer.write.call_args.args[0].to_dict()["attributes"] == { + "python_analysis_version": util.get_analysis_version(), + "python_runtime_version": platform.python_version(), + "extractor_version": util.VERSION, + "extractor_flags": "quiet", + } + logger.warning.assert_not_called() + + +def test_write_extractor_telemetry_handles_io_error(mocker): + diagnostics_writer = mocker.Mock() + diagnostics_writer.write.side_effect = OSError("write failed") + logger = mocker.Mock() + + worker._write_extractor_telemetry(diagnostics_writer, logger, []) + + logger.warning.assert_called_once_with( + "Failed to write extractor telemetry: %s", diagnostics_writer.write.side_effect + ) + + +def test_write_parser_statistics_telemetry(mocker): + diagnostics_writer = mocker.Mock() + diagnostics_writer.parser_statistics.return_value = (1, 1) + logger = mocker.Mock() + + worker._write_parser_statistics_telemetry(diagnostics_writer, logger) + + diagnostics_writer.write.assert_called_once() + assert diagnostics_writer.write.call_args.args[0].to_dict()["attributes"] == { + "old_parser_file_count": 1, + "tree_sitter_parser_file_count": 1, + } + logger.warning.assert_not_called() + + +def test_does_not_write_empty_parser_statistics_telemetry(mocker): + diagnostics_writer = mocker.Mock() + diagnostics_writer.parser_statistics.return_value = (0, 0) + logger = mocker.Mock() + + worker._write_parser_statistics_telemetry(diagnostics_writer, logger) + + diagnostics_writer.write.assert_not_called() + logger.warning.assert_not_called() + + +def test_records_old_parser_usage_once(mocker, monkeypatch): + monkeypatch.delenv("CODEQL_PYTHON_DISABLE_OLD_PARSER", raising=False) + monkeypatch.delenv("CODEQL_PYTHON_DISABLE_TSG_PARSER", raising=False) + old_ast = object() + mocker.patch("semmle.python.parser.parse", return_value=old_ast) + diagnostics_writer = worker.DiagnosticsWriter(1) + module = PythonSourceModule( + None, + "test.py", + mocker.Mock(), + diagnostics_writer, + bytes_source=b"x = 1\n", + ) + + parsed_ast = module.py_ast + # Access the cached AST again to verify that it is not counted twice. + _ = module.py_ast + + assert parsed_ast is old_ast + assert diagnostics_writer.parser_statistics() == (1, 0) + + +def test_records_tree_sitter_parser_usage_once(mocker, monkeypatch): + monkeypatch.delenv("CODEQL_PYTHON_DISABLE_OLD_PARSER", raising=False) + monkeypatch.delenv("CODEQL_PYTHON_DISABLE_TSG_PARSER", raising=False) + tree_sitter_ast = object() + mocker.patch("semmle.python.parser.parse", side_effect=SyntaxError("old parser failed")) + mocker.patch( + "semmle.python.parser.tsg_parser.parse", return_value=tree_sitter_ast + ) + diagnostics_writer = worker.DiagnosticsWriter(1) + module = PythonSourceModule( + None, + "test.py", + mocker.Mock(), + diagnostics_writer, + bytes_source=b"x = 1\n", + ) + + parsed_ast = module.py_ast + # Access the cached AST again to verify that it is not counted twice. + _ = module.py_ast + + assert parsed_ast is tree_sitter_ast + assert diagnostics_writer.parser_statistics() == (0, 1) diff --git a/python/ql/lib/qlpack.yml b/python/ql/lib/qlpack.yml index 715a4f61bfc8..ad9d7605202a 100644 --- a/python/ql/lib/qlpack.yml +++ b/python/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/python-all -version: 7.2.5 +version: 7.2.6-dev groups: python dbscheme: semmlecode.python.dbscheme extractor: python diff --git a/python/ql/src/qlpack.yml b/python/ql/src/qlpack.yml index d48bc1e8e7cc..70ce9eeef942 100644 --- a/python/ql/src/qlpack.yml +++ b/python/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/python-queries -version: 1.8.10 +version: 1.8.11-dev groups: - python - queries diff --git a/ruby/ql/lib/qlpack.yml b/ruby/ql/lib/qlpack.yml index 2c52daba2b15..f38a7120a31f 100644 --- a/ruby/ql/lib/qlpack.yml +++ b/ruby/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/ruby-all -version: 7.0.0 +version: 7.0.1-dev groups: ruby extractor: ruby dbscheme: ruby.dbscheme diff --git a/ruby/ql/src/qlpack.yml b/ruby/ql/src/qlpack.yml index a215aec72b83..d532c03248a0 100644 --- a/ruby/ql/src/qlpack.yml +++ b/ruby/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/ruby-queries -version: 1.6.10 +version: 1.6.11-dev groups: - ruby - queries diff --git a/rust/ql/lib/qlpack.yml b/rust/ql/lib/qlpack.yml index dc1059ff64c5..bb31a181ec16 100644 --- a/rust/ql/lib/qlpack.yml +++ b/rust/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/rust-all -version: 0.2.21 +version: 0.2.22-dev groups: rust extractor: rust dbscheme: rust.dbscheme diff --git a/rust/ql/src/qlpack.yml b/rust/ql/src/qlpack.yml index 2e13de282d58..dc4e53aeaf81 100644 --- a/rust/ql/src/qlpack.yml +++ b/rust/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/rust-queries -version: 0.1.42 +version: 0.1.43-dev groups: - rust - queries diff --git a/shared/concepts/qlpack.yml b/shared/concepts/qlpack.yml index f5005081b57a..677235474b91 100644 --- a/shared/concepts/qlpack.yml +++ b/shared/concepts/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/concepts -version: 0.0.31 +version: 0.0.32-dev groups: shared library: true dependencies: diff --git a/shared/controlflow/qlpack.yml b/shared/controlflow/qlpack.yml index fd5bfcb78586..66ae31f61364 100644 --- a/shared/controlflow/qlpack.yml +++ b/shared/controlflow/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/controlflow -version: 2.0.41 +version: 2.0.42-dev groups: shared library: true dependencies: diff --git a/shared/dataflow/qlpack.yml b/shared/dataflow/qlpack.yml index 09e851c73c93..134205735fe7 100644 --- a/shared/dataflow/qlpack.yml +++ b/shared/dataflow/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/dataflow -version: 2.1.13 +version: 2.1.14-dev groups: shared library: true dependencies: diff --git a/shared/mad/qlpack.yml b/shared/mad/qlpack.yml index e350a3a2ddbd..783f372bc478 100644 --- a/shared/mad/qlpack.yml +++ b/shared/mad/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/mad -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true dependencies: diff --git a/shared/namebinding/qlpack.yml b/shared/namebinding/qlpack.yml index f18af62921c4..af4358254944 100644 --- a/shared/namebinding/qlpack.yml +++ b/shared/namebinding/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/namebinding -version: 0.0.6 +version: 0.0.7-dev groups: shared library: true dependencies: diff --git a/shared/quantum/qlpack.yml b/shared/quantum/qlpack.yml index 83384e926bcb..044e243531fd 100644 --- a/shared/quantum/qlpack.yml +++ b/shared/quantum/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/quantum -version: 0.0.35 +version: 0.0.36-dev groups: shared library: true dependencies: diff --git a/shared/rangeanalysis/qlpack.yml b/shared/rangeanalysis/qlpack.yml index 2b0eb43866da..6b4e8a5adede 100644 --- a/shared/rangeanalysis/qlpack.yml +++ b/shared/rangeanalysis/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/rangeanalysis -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true dependencies: diff --git a/shared/regex/qlpack.yml b/shared/regex/qlpack.yml index d4ad24634326..ad414ca86d3d 100644 --- a/shared/regex/qlpack.yml +++ b/shared/regex/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/regex -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true dependencies: diff --git a/shared/ssa/qlpack.yml b/shared/ssa/qlpack.yml index 92b8603a9d6e..07e6041a5142 100644 --- a/shared/ssa/qlpack.yml +++ b/shared/ssa/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/ssa -version: 2.0.33 +version: 2.0.34-dev groups: shared library: true dependencies: diff --git a/shared/threat-models/qlpack.yml b/shared/threat-models/qlpack.yml index a51fdda87b7f..84c172ce3f25 100644 --- a/shared/threat-models/qlpack.yml +++ b/shared/threat-models/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/threat-models -version: 1.0.57 +version: 1.0.58-dev library: true groups: shared dataExtensions: diff --git a/shared/tree-sitter-extractor/src/generator/ql.rs b/shared/tree-sitter-extractor/src/generator/ql.rs index f114e251af21..2d91d6c66f67 100644 --- a/shared/tree-sitter-extractor/src/generator/ql.rs +++ b/shared/tree-sitter-extractor/src/generator/ql.rs @@ -109,7 +109,7 @@ impl fmt::Display for Class<'_> { is_final: false, return_type: None, formal_parameters: vec![], - body: charpred.clone(), + body: Some(charpred.clone()), overlay: None, } )?; @@ -307,7 +307,9 @@ pub struct Predicate<'a> { pub is_final: bool, pub return_type: Option>, pub formal_parameters: Vec>, - pub body: Expression<'a>, + /// The body of the predicate, or `None` if this is an `abstract` + /// predicate declaration with no body. + pub body: Option>, pub overlay: Option, } @@ -330,6 +332,9 @@ impl fmt::Display for Predicate<'_> { if self.is_final { write!(f, "final ")?; } + if self.body.is_none() { + write!(f, "abstract ")?; + } if self.overridden { write!(f, "override ")?; } @@ -344,7 +349,10 @@ impl fmt::Display for Predicate<'_> { } write!(f, "{param}")?; } - write!(f, ") {{ {} }}", self.body)?; + match &self.body { + Some(body) => write!(f, ") {{ {body} }}")?, + None => write!(f, ");")?, + } Ok(()) } diff --git a/shared/tree-sitter-extractor/src/generator/ql_gen.rs b/shared/tree-sitter-extractor/src/generator/ql_gen.rs index 237ed9ddb968..b6f3d45f4b12 100644 --- a/shared/tree-sitter-extractor/src/generator/ql_gen.rs +++ b/shared/tree-sitter-extractor/src/generator/ql_gen.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::collections::BTreeSet; use crate::{generator::ql, node_types}; @@ -20,14 +21,14 @@ pub fn create_ast_node_class<'a>( is_final: false, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: ql::Expression::Equals( + body: Some(ql::Expression::Equals( Box::new(ql::Expression::Var("result")), Box::new(ql::Expression::Dot( Box::new(ql::Expression::Var("this")), "getAPrimaryQlClass", vec![], )), - ), + )), overlay: None, }; let get_location = ql::Predicate { @@ -38,10 +39,10 @@ pub fn create_ast_node_class<'a>( is_final: true, return_type: Some(ql::Type::Normal("L::Location")), formal_parameters: vec![], - body: ql::Expression::Pred( + body: Some(ql::Expression::Pred( node_location_table, vec![ql::Expression::Var("this"), ql::Expression::Var("result")], - ), + )), overlay: None, }; let get_a_field_or_child = create_none_predicate( @@ -58,14 +59,14 @@ pub fn create_ast_node_class<'a>( is_final: true, return_type: Some(ql::Type::Facade("AstNode")), formal_parameters: vec![], - body: ql::Expression::Pred( + body: Some(ql::Expression::Pred( node_parent_table, vec![ ql::Expression::Var("this"), ql::Expression::Var("result"), ql::Expression::Var("_"), ], - ), + )), overlay: None, }; let get_parent_index = ql::Predicate { @@ -78,14 +79,14 @@ pub fn create_ast_node_class<'a>( is_final: true, return_type: Some(ql::Type::Int), formal_parameters: vec![], - body: ql::Expression::Pred( + body: Some(ql::Expression::Pred( node_parent_table, vec![ ql::Expression::Var("this"), ql::Expression::Var("_"), ql::Expression::Var("result"), ], - ), + )), overlay: None, }; let get_a_primary_ql_class = ql::Predicate { @@ -98,10 +99,10 @@ pub fn create_ast_node_class<'a>( is_final: false, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: ql::Expression::Equals( + body: Some(ql::Expression::Equals( Box::new(ql::Expression::Var("result")), Box::new(ql::Expression::String("???")), - ), + )), overlay: None, }; let get_primary_ql_classes = ql::Predicate { @@ -116,7 +117,7 @@ pub fn create_ast_node_class<'a>( is_final: false, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: ql::Expression::Equals( + body: Some(ql::Expression::Equals( Box::new(ql::Expression::Var("result")), Box::new(ql::Expression::Aggregate { name: "concat", @@ -129,7 +130,7 @@ pub fn create_ast_node_class<'a>( )), second_expr: Some(Box::new(ql::Expression::String(","))), }), - ), + )), overlay: None, }; ql::Class { @@ -163,7 +164,12 @@ pub fn create_token_class<'a>(token_type: &'a str, tokeninfo: &'a str) -> ql::Cl is_final: true, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: create_get_field_expr_for_column_storage("result", tokeninfo, 1, tokeninfo_arity), + body: Some(create_get_field_expr_for_column_storage( + "result", + tokeninfo, + 1, + tokeninfo_arity, + )), overlay: None, }; let to_string = ql::Predicate { @@ -176,14 +182,14 @@ pub fn create_token_class<'a>(token_type: &'a str, tokeninfo: &'a str) -> ql::Cl is_final: true, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: ql::Expression::Equals( + body: Some(ql::Expression::Equals( Box::new(ql::Expression::Var("result")), Box::new(ql::Expression::Dot( Box::new(ql::Expression::Var("this")), "getValue", vec![], )), - ), + )), overlay: None, }; ql::Class { @@ -223,12 +229,12 @@ pub fn create_trivia_token_class<'a>( is_final: true, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: create_get_field_expr_for_column_storage( + body: Some(create_get_field_expr_for_column_storage( "result", trivia_tokeninfo, 1, trivia_tokeninfo_arity, - ), + )), overlay: None, }; let to_string = ql::Predicate { @@ -241,14 +247,14 @@ pub fn create_trivia_token_class<'a>( is_final: true, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: ql::Expression::Equals( + body: Some(ql::Expression::Equals( Box::new(ql::Expression::Var("result")), Box::new(ql::Expression::Dot( Box::new(ql::Expression::Var("this")), "getValue", vec![], )), - ), + )), overlay: None, }; ql::Class { @@ -306,7 +312,7 @@ fn create_none_predicate<'a>( is_final: false, return_type, formal_parameters: Vec::new(), - body: ql::Expression::Pred("none", vec![]), + body: Some(ql::Expression::Pred("none", vec![])), overlay: None, } } @@ -324,10 +330,10 @@ fn create_get_a_primary_ql_class(class_name: &str, is_final: bool) -> ql::Predic is_final, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: ql::Expression::Equals( + body: Some(ql::Expression::Equals( Box::new(ql::Expression::Var("result")), Box::new(ql::Expression::String(class_name)), - ), + )), overlay: None, } } @@ -342,13 +348,13 @@ pub fn create_is_overlay_predicate() -> ql::Predicate<'static> { return_type: None, overlay: Some(ql::OverlayAnnotation::Local), formal_parameters: vec![], - body: ql::Expression::Pred( + body: Some(ql::Expression::Pred( "databaseMetadata", vec![ ql::Expression::String("isOverlay"), ql::Expression::String("true"), ], - ), + )), } } @@ -368,7 +374,7 @@ pub fn create_get_node_file_predicate<'a>( name: "node", param_type: ql::Type::At(ast_node_name), }], - body: ql::Expression::Aggregate { + body: Some(ql::Expression::Aggregate { name: "exists", vars: vec![ql::FormalParameter { name: "loc", @@ -390,7 +396,7 @@ pub fn create_get_node_file_predicate<'a>( ], )), second_expr: None, - }, + }), } } @@ -415,7 +421,7 @@ pub fn create_discardable_ast_node_predicate(ast_node_name: &str) -> ql::Predica param_type: ql::Type::At(ast_node_name), }, ], - body: ql::Expression::And(vec![ + body: Some(ql::Expression::And(vec![ ql::Expression::Negation(Box::new(ql::Expression::Pred("isOverlay", vec![]))), ql::Expression::Equals( Box::new(ql::Expression::Var("file")), @@ -424,7 +430,7 @@ pub fn create_discardable_ast_node_predicate(ast_node_name: &str) -> ql::Predica vec![ql::Expression::Var("node")], )), ), - ]), + ])), } } @@ -444,7 +450,7 @@ pub fn create_discard_ast_node_predicate(ast_node_name: &str) -> ql::Predicate<' name: "node", param_type: ql::Type::At(ast_node_name), }], - body: ql::Expression::Aggregate { + body: Some(ql::Expression::Aggregate { name: "exists", vars: vec![ ql::FormalParameter { @@ -468,7 +474,7 @@ pub fn create_discard_ast_node_predicate(ast_node_name: &str) -> ql::Predicate<' ql::Expression::Pred("overlayChangedFiles", vec![ql::Expression::Var("path")]), ])), second_expr: None, - }, + }), } } @@ -493,7 +499,7 @@ pub fn create_discardable_location_predicate() -> ql::Predicate<'static> { param_type: ql::Type::At("location_default"), }, ], - body: ql::Expression::And(vec![ + body: Some(ql::Expression::And(vec![ ql::Expression::Negation(Box::new(ql::Expression::Pred("isOverlay", vec![]))), ql::Expression::Pred( "locations_default", @@ -506,7 +512,7 @@ pub fn create_discardable_location_predicate() -> ql::Predicate<'static> { ql::Expression::Var("_"), ], ), - ]), + ])), } } @@ -529,7 +535,7 @@ pub fn create_discard_location_predicate() -> ql::Predicate<'static> { name: "loc", param_type: ql::Type::At("location_default"), }], - body: ql::Expression::Aggregate { + body: Some(ql::Expression::Aggregate { name: "exists", vars: vec![ ql::FormalParameter { @@ -553,7 +559,7 @@ pub fn create_discard_location_predicate() -> ql::Predicate<'static> { ql::Expression::Pred("overlayChangedFiles", vec![ql::Expression::Var("path")]), ])), second_expr: None, - }, + }), } } @@ -760,7 +766,7 @@ fn create_field_getters<'a>( is_final: true, return_type: return_type.clone(), formal_parameters, - body, + body: Some(body), overlay: None, }]; @@ -773,14 +779,14 @@ fn create_field_getters<'a>( is_final: true, return_type, formal_parameters: vec![], - body: ql::Expression::Equals( + body: Some(ql::Expression::Equals( Box::new(ql::Expression::Var("result")), Box::new(ql::Expression::Dot( Box::new(ql::Expression::Var("this")), &field.getter_name, vec![ql::Expression::Var("_")], )), - ), + )), overlay: None, }); } @@ -828,6 +834,89 @@ fn class_supertypes<'a>( supertypes } +/// Returns whether `a` and `b` have the same signature, i.e. the same name, +/// return type, and formal parameters. Predicates with the same signature can +/// override one another. +fn same_predicate_signature(a: &ql::Predicate, b: &ql::Predicate) -> bool { + a.name == b.name && a.return_type == b.return_type && a.formal_parameters == b.formal_parameters +} + +/// Computes, for each tree-sitter supertype (union) node, the list of +/// predicates that are guaranteed to be defined identically (in terms of +/// name, return type, and formal parameters, though not necessarily body) by +/// every one of its members. These are the predicates that can be hoisted to +/// an `abstract` predicate on the union's class, with the corresponding +/// predicates on its members becoming `override`s. +/// +/// The result for a given node is memoized in `cache` (keyed by its QL class +/// name), and also used to answer the query for any other node that +/// (directly, or transitively through further supertypes) has that node as a +/// member. The same cache also serves as the answer to "what does the class +/// named X expose?", used by `is_predicate_inherited`. +fn compute_exposed_predicates<'a, 'b>( + type_name: &'a node_types::TypeName, + nodes: &'a node_types::NodeTypeMap, + field_predicates: &BTreeMap<&node_types::TypeName, Vec>>, + cache: &'b mut BTreeMap<&'a str, Vec>>, +) -> &'b Vec> { + let node = nodes.get(type_name); + let class_name = node.map_or(type_name.kind.as_str(), |node| node.ql_class_name.as_str()); + if !cache.contains_key(class_name) { + // Supertype declarations that recursively refer to themselves are a mistake, but we don't + // want to cause infinite recursion, so we insert a temporary sentinel. + cache.insert(class_name, Vec::new()); + let exposed = match node.map(|node| &node.kind) { + Some(node_types::EntryKind::Table { .. }) => { + field_predicates.get(type_name).cloned().unwrap_or_default() + } + Some(node_types::EntryKind::Union { members }) => { + let mut members = members.iter(); + let mut common = match members.next() { + Some(first) => { + compute_exposed_predicates(first, nodes, field_predicates, cache).clone() + } + None => Vec::new(), + }; + for member in members { + let member_predicates = + compute_exposed_predicates(member, nodes, field_predicates, cache); + common.retain(|predicate| { + member_predicates + .iter() + .any(|other| same_predicate_signature(predicate, other)) + }); + } + common + } + Some(node_types::EntryKind::Token { .. }) | None => Vec::new(), + }; + cache.insert(class_name, exposed); + } + cache.get(class_name).unwrap() +} + +/// Returns whether `predicate` (declared, or about to be declared, on the +/// class for `type_name`) is already exposed by one of `type_name`'s direct +/// supertypes, and therefore must be marked as an `override` (for a concrete +/// predicate) or can be omitted entirely (for an `abstract` one, since it's +/// already inherited). +fn is_predicate_inherited( + predicate: &ql::Predicate, + type_name: &node_types::TypeName, + direct_supertypes: &BTreeMap>, + exposed_predicates: &BTreeMap<&str, Vec>, +) -> bool { + direct_supertypes.get(type_name).is_some_and(|supertypes| { + supertypes.iter().any(|supertype| { + exposed_predicates.get(supertype).is_some_and(|predicates| { + predicates + .iter() + .any(|other| same_predicate_signature(predicate, other)) + }) + }) + }) +} + /// Converts the given node types into CodeQL classes wrapping the dbscheme. pub fn convert_nodes(nodes: &node_types::NodeTypeMap) -> Vec> { let mut classes = Vec::new(); @@ -841,6 +930,71 @@ pub fn convert_nodes(nodes: &node_types::NodeTypeMap) -> Vec> { } } + // First, compute the field-getter predicates (and the expressions used by + // `getAFieldOrChild`) for every table node, without yet knowing whether + // any of them will need to be marked `override`. These are needed both + // to build the final classes below, and to figure out which fields are + // shared identically by all the members of a supertype. + let mut field_predicates: BTreeMap<&node_types::TypeName, Vec>> = + BTreeMap::new(); + let mut get_child_exprs: BTreeMap<&node_types::TypeName, Vec>> = + BTreeMap::new(); + for (type_name, node) in nodes { + if let node_types::EntryKind::Table { + name: main_table_name, + fields, + } = &node.kind + { + if fields.is_empty() { + panic!("Encountered node '{}' with no fields", type_name.kind); + } + + // Count how many columns there will be in the main table. There + // will be one for the id, plus one for each field that's stored + // as a column. + let main_table_arity = 1 + fields + .iter() + .filter(|&f| matches!(f.storage, node_types::Storage::Column { .. })) + .count(); + + let mut main_table_column_index: usize = 0; + let mut predicates = Vec::new(); + let mut exprs = Vec::new(); + for field in fields { + let (get_preds, get_child_expr) = create_field_getters( + main_table_name, + main_table_arity, + &mut main_table_column_index, + field, + nodes, + ); + predicates.extend(get_preds); + if let Some(get_child_expr) = get_child_expr { + exprs.push(get_child_expr) + } + } + field_predicates.insert(type_name, predicates); + get_child_exprs.insert(type_name, exprs); + } + } + + // Next, for every supertype (union) node, compute the predicates that are + // guaranteed to be defined identically (in name, return type, and formal + // parameters) by every one of its members. Such predicates can be hoisted + // to an `abstract` predicate on the supertype's class, with the + // corresponding predicates on its members becoming `override`s. + let mut exposed_predicates: BTreeMap<&str, Vec>> = BTreeMap::new(); + for (type_name, node) in nodes { + if let node_types::EntryKind::Union { .. } = &node.kind { + compute_exposed_predicates( + type_name, + nodes, + &field_predicates, + &mut exposed_predicates, + ); + } + } + for (type_name, node) in nodes { match &node.kind { node_types::EntryKind::Token { kind_id: _ } => { @@ -865,7 +1019,26 @@ pub fn convert_nodes(nodes: &node_types::NodeTypeMap) -> Vec> { } node_types::EntryKind::Union { members: _ } => { // It's a tree-sitter supertype node, so we're wrapping a dbscheme - // union type. + // union type. Any predicate that's identically defined by every + // member becomes an `abstract` predicate here. + let predicates = exposed_predicates + .get(node.ql_class_name.as_str()) + .cloned() + .unwrap_or_default() + .into_iter() + .map(|predicate| ql::Predicate { + overridden: is_predicate_inherited( + &predicate, + type_name, + &direct_supertypes, + &exposed_predicates, + ), + is_private: false, + is_final: false, + body: None, + ..predicate + }) + .collect(); classes.push(ql::TopLevel::Class(ql::Class { qldoc: None, name: &node.ql_class_name, @@ -879,25 +1052,10 @@ pub fn convert_nodes(nodes: &node_types::NodeTypeMap) -> Vec> { &direct_supertypes, ), characteristic_predicate: None, - predicates: vec![], + predicates, })); } - node_types::EntryKind::Table { - name: main_table_name, - fields, - } => { - if fields.is_empty() { - panic!("Encountered node '{}' with no fields", type_name.kind); - } - - // Count how many columns there will be in the main table. There - // will be one for the id, plus one for each field that's stored - // as a column. - let main_table_arity = 1 + fields - .iter() - .filter(|&f| matches!(f.storage, node_types::Storage::Column { .. })) - .count(); - + node_types::EntryKind::Table { .. } => { let main_class_name = &node.ql_class_name; let mut main_class = ql::Class { qldoc: Some(format!("A class representing `{}` nodes.", type_name.kind)), @@ -915,26 +1073,30 @@ pub fn convert_nodes(nodes: &node_types::NodeTypeMap) -> Vec> { predicates: vec![create_get_a_primary_ql_class(main_class_name, true)], }; - let mut main_table_column_index: usize = 0; - let mut get_child_exprs: Vec = Vec::new(); - - // Iterate through the fields, creating: - // - classes to wrap union types if fields need them, - // - predicates to access the fields, - // - the QL expressions to access the fields that will be part of getAFieldOrChild. - for field in fields { - let (get_preds, get_child_expr) = create_field_getters( - main_table_name, - main_table_arity, - &mut main_table_column_index, - field, - nodes, - ); - main_class.predicates.extend(get_preds); - if let Some(get_child_expr) = get_child_expr { - get_child_exprs.push(get_child_expr) - } - } + // A field getter that's identically defined (in signature) by + // every member of one of this node's direct supertypes is an + // override of the corresponding `abstract` predicate declared + // there. + main_class.predicates.extend( + field_predicates + .get(type_name) + .cloned() + .unwrap_or_default() + .into_iter() + .map(|predicate| { + let overridden = predicate.overridden + || is_predicate_inherited( + &predicate, + type_name, + &direct_supertypes, + &exposed_predicates, + ); + ql::Predicate { + overridden, + ..predicate + } + }), + ); main_class.predicates.push(ql::Predicate { qldoc: Some(String::from("Gets a field or child node of this node.")), @@ -944,7 +1106,9 @@ pub fn convert_nodes(nodes: &node_types::NodeTypeMap) -> Vec> { is_final: true, return_type: Some(ql::Type::Facade("AstNode")), formal_parameters: vec![], - body: ql::Expression::Or(get_child_exprs), + body: Some(ql::Expression::Or( + get_child_exprs.get(type_name).cloned().unwrap_or_default(), + )), overlay: None, }); @@ -1038,7 +1202,7 @@ pub fn create_print_ast_module(nodes: &node_types::NodeTypeMap) -> ql::TopLevel< param_type: ql::Type::Int, }, ], - body: ql::Expression::Or(disjuncts), + body: Some(ql::Expression::Or(disjuncts)), overlay: None, }; diff --git a/shared/tutorial/qlpack.yml b/shared/tutorial/qlpack.yml index 2c2dde8c1793..fa374cda8f72 100644 --- a/shared/tutorial/qlpack.yml +++ b/shared/tutorial/qlpack.yml @@ -1,7 +1,7 @@ name: codeql/tutorial description: Library for the CodeQL detective tutorials, helping new users learn to write CodeQL queries. -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true warnOnImplicitThis: true diff --git a/shared/typeflow/qlpack.yml b/shared/typeflow/qlpack.yml index 6ea63f761cdd..4dd13d3aad94 100644 --- a/shared/typeflow/qlpack.yml +++ b/shared/typeflow/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typeflow -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true dependencies: diff --git a/shared/typeinference/qlpack.yml b/shared/typeinference/qlpack.yml index 5cabb023ce1e..68db81a4851b 100644 --- a/shared/typeinference/qlpack.yml +++ b/shared/typeinference/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typeinference -version: 0.0.38 +version: 0.0.39-dev groups: shared library: true dependencies: diff --git a/shared/typetracking/qlpack.yml b/shared/typetracking/qlpack.yml index 854d8bae6da0..9b52023e6343 100644 --- a/shared/typetracking/qlpack.yml +++ b/shared/typetracking/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typetracking -version: 2.0.41 +version: 2.0.42-dev groups: shared library: true dependencies: diff --git a/shared/typos/qlpack.yml b/shared/typos/qlpack.yml index 712073d7668a..ade80e7678f0 100644 --- a/shared/typos/qlpack.yml +++ b/shared/typos/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typos -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true warnOnImplicitThis: true diff --git a/shared/util/qlpack.yml b/shared/util/qlpack.yml index 76bb1b6957a2..0a61f3903159 100644 --- a/shared/util/qlpack.yml +++ b/shared/util/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/util -version: 2.0.44 +version: 2.0.45-dev groups: shared library: true dependencies: null diff --git a/shared/xml/qlpack.yml b/shared/xml/qlpack.yml index c9251eb88c4d..133a3b9dac86 100644 --- a/shared/xml/qlpack.yml +++ b/shared/xml/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/xml -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true dependencies: diff --git a/shared/yaml/qlpack.yml b/shared/yaml/qlpack.yml index ccc990b27d02..14b67df2f8d1 100644 --- a/shared/yaml/qlpack.yml +++ b/shared/yaml/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/yaml -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true warnOnImplicitThis: true diff --git a/swift/ql/lib/qlpack.yml b/swift/ql/lib/qlpack.yml index dd31b62e1481..9806fded2032 100644 --- a/swift/ql/lib/qlpack.yml +++ b/swift/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/swift-all -version: 6.8.3 +version: 6.8.4-dev groups: swift extractor: swift dbscheme: swift.dbscheme diff --git a/swift/ql/src/qlpack.yml b/swift/ql/src/qlpack.yml index fd6ec0e549bc..aa419cb33887 100644 --- a/swift/ql/src/qlpack.yml +++ b/swift/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/swift-queries -version: 1.3.10 +version: 1.3.11-dev groups: - swift - queries diff --git a/unified/ql/lib/codeql/unified/internal/Ast.qll b/unified/ql/lib/codeql/unified/internal/Ast.qll index cf927e257a9c..d47bd07d142e 100644 --- a/unified/ql/lib/codeql/unified/internal/Ast.qll +++ b/unified/ql/lib/codeql/unified/internal/Ast.qll @@ -104,7 +104,7 @@ module Unified { final F::AccessorKind getAccessorKind() { unified_accessor_declaration_def(this, result, _) } /** Gets the node corresponding to the field `body`. */ - final F::Block getBody() { unified_accessor_declaration_body(this, result) } + final override F::Block getBody() { unified_accessor_declaration_body(this, result) } /** Gets the node corresponding to the field `modifier`. */ final F::Modifier getModifier(int i) { unified_accessor_declaration_modifier(this, i, result) } @@ -373,7 +373,10 @@ module Unified { } } - class Callable extends @unified_callable, F::AstNode { } + class Callable extends @unified_callable, F::AstNode { + /** Gets the node corresponding to the field `body`. */ + abstract F::Block getBody(); + } /** A class representing `catch_clause` nodes. */ class CatchClause extends @unified_catch_clause, F::AstNode { @@ -512,7 +515,7 @@ module Unified { final override string getAPrimaryQlClass() { result = "ConstructorDeclaration" } /** Gets the node corresponding to the field `body`. */ - final F::Block getBody() { unified_constructor_declaration_def(this, result) } + final override F::Block getBody() { unified_constructor_declaration_def(this, result) } /** Gets the node corresponding to the field `modifier`. */ final F::Modifier getModifier(int i) { @@ -592,7 +595,7 @@ module Unified { final override string getAPrimaryQlClass() { result = "DestructorDeclaration" } /** Gets the node corresponding to the field `body`. */ - final F::Block getBody() { unified_destructor_declaration_def(this, result) } + final override F::Block getBody() { unified_destructor_declaration_def(this, result) } /** Gets the node corresponding to the field `modifier`. */ final F::Modifier getModifier(int i) { @@ -727,7 +730,7 @@ module Unified { final override string getAPrimaryQlClass() { result = "FunctionDeclaration" } /** Gets the node corresponding to the field `body`. */ - final F::Block getBody() { unified_function_declaration_body(this, result) } + final override F::Block getBody() { unified_function_declaration_body(this, result) } /** Gets the node corresponding to the field `modifier`. */ final F::Modifier getModifier(int i) { unified_function_declaration_modifier(this, i, result) } @@ -783,7 +786,7 @@ module Unified { final override string getAPrimaryQlClass() { result = "FunctionExpr" } /** Gets the node corresponding to the field `body`. */ - final F::Block getBody() { unified_function_expr_def(this, result) } + final override F::Block getBody() { unified_function_expr_def(this, result) } /** Gets the node corresponding to the field `capture_declaration`. */ final F::VariableDeclaration getCaptureDeclaration(int i) { @@ -958,7 +961,7 @@ module Unified { final override string getAPrimaryQlClass() { result = "InitializerDeclaration" } /** Gets the node corresponding to the field `body`. */ - final F::Block getBody() { unified_initializer_declaration_def(this, result) } + final override F::Block getBody() { unified_initializer_declaration_def(this, result) } /** Gets the node corresponding to the field `modifier`. */ final F::Modifier getModifier(int i) { @@ -1359,7 +1362,7 @@ module Unified { final override string getAPrimaryQlClass() { result = "TopLevel" } /** Gets the node corresponding to the field `body`. */ - final F::Block getBody() { unified_top_level_def(this, result) } + final override F::Block getBody() { unified_top_level_def(this, result) } /** Gets a field or child node of this node. */ final override F::AstNode getAFieldOrChild() { unified_top_level_def(this, result) } diff --git a/unified/ql/lib/codeql/unified/internal/ControlFlowGraph.qll b/unified/ql/lib/codeql/unified/internal/ControlFlowGraph.qll index 15db0eb3836e..47bda0e16dd3 100644 --- a/unified/ql/lib/codeql/unified/internal/ControlFlowGraph.qll +++ b/unified/ql/lib/codeql/unified/internal/ControlFlowGraph.qll @@ -46,15 +46,7 @@ private module Ast implements AstSig { class Callable = U::Callable; - AstNode callableGetBody(Callable c) { - result = c.(AccessorDeclaration).getBody() or - result = c.(ConstructorDeclaration).getBody() or - result = c.(DestructorDeclaration).getBody() or - result = c.(FunctionDeclaration).getBody() or - result = c.(FunctionExpr).getBody() or - result = c.(InitializerDeclaration).getBody() or - result = c.(TopLevel).getBody() - } + AstNode callableGetBody(Callable c) { result = c.getBody() } class Parameter extends U::Parameter { Expr getDefaultValue() { result = super.getDefault() }