From 8701ce9f7e64bd26f4194fab75c7f6324be2c661 Mon Sep 17 00:00:00 2001 From: AkshayK Date: Thu, 27 Aug 2026 15:08:27 -0400 Subject: [PATCH 01/19] cpp: model BDE bdlbb::Blob byte-buffer taint flow Add flow summaries for the BDE segmented byte buffer BloombergLP::bdlbb::Blob so taint reaches a blob's payload bytes: - Accessor chain: Blob::buffer taints the returned BlobBuffer, and BlobBuffer::data/buffer taint the bytes. - bdlbb::BlobUtil::copy and getContiguousRangeOrCopy propagate taint between a blob and a flat buffer in both directions. This unblocks blob-carried sources such as bmqa::Message::getData, whose payload was previously stranded on the opaque Blob object. Not a duplicate; the bdlbb namespace had no coverage. Verified with a BloombergLP::bdlbb-shaped stub in the dataflow external-models harness. --- .../2026-08-27-bdlbb-blob-models.md | 4 + cpp/ql/lib/ext/bdlbb.model.yml | 20 +++++ .../dataflow/external-models/bdlbb.cpp | 83 +++++++++++++++++++ .../dataflow/external-models/flow.expected | 64 +++++++++++++- .../dataflow/external-models/steps.expected | 11 +++ .../external-models/validatemodels.expected | 3 + 6 files changed, 183 insertions(+), 2 deletions(-) create mode 100644 cpp/ql/lib/change-notes/2026-08-27-bdlbb-blob-models.md create mode 100644 cpp/ql/lib/ext/bdlbb.model.yml create mode 100644 cpp/ql/test/library-tests/dataflow/external-models/bdlbb.cpp diff --git a/cpp/ql/lib/change-notes/2026-08-27-bdlbb-blob-models.md b/cpp/ql/lib/change-notes/2026-08-27-bdlbb-blob-models.md new file mode 100644 index 000000000000..f1db9d3c3116 --- /dev/null +++ b/cpp/ql/lib/change-notes/2026-08-27-bdlbb-blob-models.md @@ -0,0 +1,4 @@ +--- +category: minorAnalysis +--- +* Added flow summaries for the BDE `bdlbb::Blob` segmented byte buffer (`BloombergLP::bdlbb`). Taint now flows from a blob to its bytes through the `Blob::buffer`/`BlobBuffer::data` accessor chain and through the `bdlbb::BlobUtil::copy` and `getContiguousRangeOrCopy` helpers, so a blob populated from untrusted input (for example a BlazingMQ message body read via `bmqa::Message::getData`) is tracked into the payload bytes. diff --git a/cpp/ql/lib/ext/bdlbb.model.yml b/cpp/ql/lib/ext/bdlbb.model.yml new file mode 100644 index 000000000000..135f6c631e83 --- /dev/null +++ b/cpp/ql/lib/ext/bdlbb.model.yml @@ -0,0 +1,20 @@ +# Model of the BDE bdlbb::Blob segmented byte buffer (BloombergLP::bdlbb). +# Lets taint reach a blob's payload bytes, e.g. a message body filled by bmqa::Message::getData. +extensions: + - addsTo: + pack: codeql/cpp-all + extensible: summaryModel + data: # namespace, type, subtypes, name, signature, ext, input, output, kind, provenance + # Accessor chain: a tainted blob taints its buffers, and a tainted buffer taints its bytes. + - ["BloombergLP::bdlbb", "Blob", true, "buffer", "", "", "Argument[-1]", "ReturnValue[*]", "taint", "manual"] + - ["BloombergLP::bdlbb", "BlobBuffer", true, "data", "", "", "Argument[-1]", "ReturnValue[*]", "taint", "manual"] + # BlobUtil read-out: the source blob (Argument[*1]) taints the destination buffer (and the + # returned contiguous range). + - ["BloombergLP::bdlbb", "BlobUtil", true, "copy", "(char *,const Blob &,int,int)", "", "Argument[*1]", "Argument[*0]", "taint", "manual"] + - ["BloombergLP::bdlbb", "BlobUtil", true, "getContiguousRangeOrCopy", "", "", "Argument[*1]", "Argument[*0]", "taint", "manual"] + - ["BloombergLP::bdlbb", "BlobUtil", true, "getContiguousRangeOrCopy", "", "", "Argument[*1]", "ReturnValue[*]", "taint", "manual"] + # BlobUtil write-in: the source (Argument[*2]) taints the destination blob. `copy` has two + # write-in overloads, one taking a raw byte buffer and one taking another blob as the source; + # each row pins the exact signature so the int offset/length arguments are never tainted. + - ["BloombergLP::bdlbb", "BlobUtil", true, "copy", "(Blob *,int,const char *,int)", "", "Argument[*2]", "Argument[*0]", "taint", "manual"] + - ["BloombergLP::bdlbb", "BlobUtil", true, "copy", "(Blob *,int,const Blob &,int,int)", "", "Argument[*2]", "Argument[*0]", "taint", "manual"] diff --git a/cpp/ql/test/library-tests/dataflow/external-models/bdlbb.cpp b/cpp/ql/test/library-tests/dataflow/external-models/bdlbb.cpp new file mode 100644 index 000000000000..a76174c42d46 --- /dev/null +++ b/cpp/ql/test/library-tests/dataflow/external-models/bdlbb.cpp @@ -0,0 +1,83 @@ + +// --- stub library headers --- + +namespace bsl { + typedef unsigned long size_t; + template class allocator {}; + template struct char_traits {}; + template, class Allocator = allocator > + class basic_string { + public: + basic_string(const charT* s, const Allocator& a = Allocator()); + const charT* data() const; + size_t size() const; + }; + typedef basic_string string; +} + +namespace BloombergLP { +namespace bdlbb { + class BlobBuffer { + public: + char *data() const; + }; + + class Blob { + public: + const BlobBuffer &buffer(int index) const; + }; + + struct BlobUtil { + static void copy(char *dstBuffer, const Blob &srcBlob, int position, int length); + static void copy(Blob *dstBlob, int dstOffset, const char *srcBuffer, int length); + static void copy(Blob *dstBlob, int dstOffset, const Blob &srcBlob, int srcOffset, + int length); + static char *getContiguousRangeOrCopy(char *dstBuffer, const Blob &srcBlob, int position, + int length, int alignment); + }; +} +} + +// --- test code --- + +char *source(); +void sink(char); + +// A blob populated from a tainted buffer taints the bytes read back out of it. +void test_BlobUtil_copy() { + bsl::string s(source()); + BloombergLP::bdlbb::Blob blob; + BloombergLP::bdlbb::BlobUtil::copy(&blob, 0, s.data(), s.size()); + char dst[16]; + BloombergLP::bdlbb::BlobUtil::copy(dst, blob, 0, 16); + sink(*dst); // $ ir +} + +void test_accessor_chain() { + bsl::string s(source()); + BloombergLP::bdlbb::Blob blob; + BloombergLP::bdlbb::BlobUtil::copy(&blob, 0, s.data(), s.size()); + const char *p = blob.buffer(0).data(); + sink(*p); // $ ir +} + +void test_getContiguousRangeOrCopy() { + bsl::string s(source()); + BloombergLP::bdlbb::Blob blob; + BloombergLP::bdlbb::BlobUtil::copy(&blob, 0, s.data(), s.size()); + char dst[16]; + char *r = BloombergLP::bdlbb::BlobUtil::getContiguousRangeOrCopy(dst, blob, 0, 16, 1); + sink(*r); // $ ir +} + +// A blob copied into another blob carries the taint across. +void test_BlobUtil_copy_blob_to_blob() { + bsl::string s(source()); + BloombergLP::bdlbb::Blob src; + BloombergLP::bdlbb::BlobUtil::copy(&src, 0, s.data(), s.size()); + BloombergLP::bdlbb::Blob dst; + BloombergLP::bdlbb::BlobUtil::copy(&dst, 0, src, 0, 16); + char out[16]; + BloombergLP::bdlbb::BlobUtil::copy(out, dst, 0, 16); + sink(*out); // $ ir +} diff --git a/cpp/ql/test/library-tests/dataflow/external-models/flow.expected b/cpp/ql/test/library-tests/dataflow/external-models/flow.expected index b6f5f4a4452f..8989ddfa612b 100644 --- a/cpp/ql/test/library-tests/dataflow/external-models/flow.expected +++ b/cpp/ql/test/library-tests/dataflow/external-models/flow.expected @@ -95,7 +95,13 @@ models | 94 | Summary: Azure::Core::IO; BodyStream; true; ReadToCount; ; ; Argument[-1]; Argument[*0]; taint; manual | | 95 | Summary: Azure::Core::IO; BodyStream; true; ReadToEnd; ; ; Argument[-1]; ReturnValue.Element; taint; manual | | 96 | Summary: Azure; Nullable; true; Value; ; ; Argument[-1]; ReturnValue[*]; taint; manual | -| 97 | Summary: boost::asio; ; false; buffer; ; ; Argument[*0]; ReturnValue; taint; manual | +| 97 | Summary: BloombergLP::bdlbb; Blob; true; buffer; ; ; Argument[-1]; ReturnValue[*]; taint; manual | +| 98 | Summary: BloombergLP::bdlbb; BlobBuffer; true; data; ; ; Argument[-1]; ReturnValue[*]; taint; manual | +| 99 | Summary: BloombergLP::bdlbb; BlobUtil; true; copy; (Blob *,int,const Blob &,int,int); ; Argument[*2]; Argument[*0]; taint; manual | +| 100 | Summary: BloombergLP::bdlbb; BlobUtil; true; copy; (Blob *,int,const char *,int); ; Argument[*2]; Argument[*0]; taint; manual | +| 101 | Summary: BloombergLP::bdlbb; BlobUtil; true; copy; (char *,const Blob &,int,int); ; Argument[*1]; Argument[*0]; taint; manual | +| 102 | Summary: BloombergLP::bdlbb; BlobUtil; true; getContiguousRangeOrCopy; ; ; Argument[*1]; ReturnValue[*]; taint; manual | +| 103 | Summary: boost::asio; ; false; buffer; ; ; Argument[*0]; ReturnValue; taint; manual | edges | asio_streams.cpp:87:34:87:44 | read_until output argument | asio_streams.cpp:91:7:91:17 | recv_buffer | provenance | Src:MaD:56 | | asio_streams.cpp:87:34:87:44 | read_until output argument | asio_streams.cpp:93:29:93:39 | recv_buffer | provenance | Src:MaD:56 Sink:MaD:4 | @@ -104,7 +110,7 @@ edges | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | | | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:101:7:101:17 | send_buffer | provenance | | | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:103:29:103:39 | send_buffer | provenance | Sink:MaD:4 | -| asio_streams.cpp:100:64:100:71 | *send_str | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | MaD:97 | +| asio_streams.cpp:100:64:100:71 | *send_str | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | MaD:103 | | azure.cpp:253:48:253:60 | *call to GetBodyStream | azure.cpp:257:5:257:8 | *resp | provenance | | | azure.cpp:253:48:253:60 | *call to GetBodyStream | azure.cpp:262:5:262:8 | *resp | provenance | | | azure.cpp:253:48:253:60 | *call to GetBodyStream | azure.cpp:266:38:266:41 | *resp | provenance | | @@ -144,6 +150,31 @@ edges | azure.cpp:294:38:294:53 | call to operator[] | azure.cpp:295:10:295:20 | contentType | provenance | | | azure.cpp:294:38:294:53 | call to operator[] | azure.cpp:295:10:295:20 | contentType | provenance | | | azure.cpp:295:10:295:20 | contentType | azure.cpp:295:10:295:20 | contentType | provenance | | +| bdlbb.cpp:48:16:48:23 | call to source | bdlbb.cpp:50:49:50:52 | *call to data | provenance | TaintFunction | +| bdlbb.cpp:50:37:50:41 | copy output argument | bdlbb.cpp:52:42:52:45 | *blob | provenance | | +| bdlbb.cpp:50:49:50:52 | *call to data | bdlbb.cpp:50:37:50:41 | copy output argument | provenance | MaD:100 | +| bdlbb.cpp:52:37:52:39 | copy output argument | bdlbb.cpp:53:7:53:10 | * ... | provenance | | +| bdlbb.cpp:52:42:52:45 | *blob | bdlbb.cpp:52:37:52:39 | copy output argument | provenance | MaD:101 | +| bdlbb.cpp:57:16:57:23 | call to source | bdlbb.cpp:59:49:59:52 | *call to data | provenance | TaintFunction | +| bdlbb.cpp:59:37:59:41 | copy output argument | bdlbb.cpp:60:18:60:21 | *blob | provenance | | +| bdlbb.cpp:59:49:59:52 | *call to data | bdlbb.cpp:59:37:59:41 | copy output argument | provenance | MaD:100 | +| bdlbb.cpp:60:18:60:21 | *blob | bdlbb.cpp:60:29:60:32 | *call to buffer | provenance | MaD:97 | +| bdlbb.cpp:60:18:60:38 | *call to data | bdlbb.cpp:60:18:60:38 | *call to data | provenance | | +| bdlbb.cpp:60:18:60:38 | *call to data | bdlbb.cpp:61:7:61:8 | * ... | provenance | | +| bdlbb.cpp:60:29:60:32 | *call to buffer | bdlbb.cpp:60:18:60:38 | *call to data | provenance | MaD:98 | +| bdlbb.cpp:65:16:65:23 | call to source | bdlbb.cpp:67:49:67:52 | *call to data | provenance | TaintFunction | +| bdlbb.cpp:67:37:67:41 | copy output argument | bdlbb.cpp:69:72:69:75 | *blob | provenance | | +| bdlbb.cpp:67:49:67:52 | *call to data | bdlbb.cpp:67:37:67:41 | copy output argument | provenance | MaD:100 | +| bdlbb.cpp:69:12:69:65 | *call to getContiguousRangeOrCopy | bdlbb.cpp:69:12:69:65 | *call to getContiguousRangeOrCopy | provenance | | +| bdlbb.cpp:69:12:69:65 | *call to getContiguousRangeOrCopy | bdlbb.cpp:70:7:70:8 | * ... | provenance | | +| bdlbb.cpp:69:72:69:75 | *blob | bdlbb.cpp:69:12:69:65 | *call to getContiguousRangeOrCopy | provenance | MaD:102 | +| bdlbb.cpp:75:16:75:23 | call to source | bdlbb.cpp:77:48:77:51 | *call to data | provenance | TaintFunction | +| bdlbb.cpp:77:37:77:40 | copy output argument | bdlbb.cpp:79:46:79:48 | *src | provenance | | +| bdlbb.cpp:77:48:77:51 | *call to data | bdlbb.cpp:77:37:77:40 | copy output argument | provenance | MaD:100 | +| bdlbb.cpp:79:37:79:40 | copy output argument | bdlbb.cpp:81:42:81:44 | *dst | provenance | | +| bdlbb.cpp:79:46:79:48 | *src | bdlbb.cpp:79:37:79:40 | copy output argument | provenance | MaD:99 | +| bdlbb.cpp:81:37:81:39 | copy output argument | bdlbb.cpp:82:7:82:10 | * ... | provenance | | +| bdlbb.cpp:81:42:81:44 | *dst | bdlbb.cpp:81:37:81:39 | copy output argument | provenance | MaD:101 | | test.cpp:7:47:7:52 | value2 | test.cpp:7:64:7:69 | value2 | provenance | | | test.cpp:7:64:7:69 | value2 | test.cpp:7:5:7:30 | *ymlStepGenerated_with_body | provenance | | | test.cpp:10:10:10:18 | call to ymlSource | test.cpp:10:10:10:18 | call to ymlSource | provenance | Src:MaD:48 | @@ -532,6 +563,35 @@ nodes | azure.cpp:295:10:295:20 | contentType | semmle.label | contentType | | azure.cpp:295:10:295:20 | contentType | semmle.label | contentType | | azure.cpp:295:10:295:20 | contentType | semmle.label | contentType | +| bdlbb.cpp:48:16:48:23 | call to source | semmle.label | call to source | +| bdlbb.cpp:50:37:50:41 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:50:49:50:52 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:52:37:52:39 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:52:42:52:45 | *blob | semmle.label | *blob | +| bdlbb.cpp:53:7:53:10 | * ... | semmle.label | * ... | +| bdlbb.cpp:57:16:57:23 | call to source | semmle.label | call to source | +| bdlbb.cpp:59:37:59:41 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:59:49:59:52 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:60:18:60:21 | *blob | semmle.label | *blob | +| bdlbb.cpp:60:18:60:38 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:60:18:60:38 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:60:29:60:32 | *call to buffer | semmle.label | *call to buffer | +| bdlbb.cpp:61:7:61:8 | * ... | semmle.label | * ... | +| bdlbb.cpp:65:16:65:23 | call to source | semmle.label | call to source | +| bdlbb.cpp:67:37:67:41 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:67:49:67:52 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:69:12:69:65 | *call to getContiguousRangeOrCopy | semmle.label | *call to getContiguousRangeOrCopy | +| bdlbb.cpp:69:12:69:65 | *call to getContiguousRangeOrCopy | semmle.label | *call to getContiguousRangeOrCopy | +| bdlbb.cpp:69:72:69:75 | *blob | semmle.label | *blob | +| bdlbb.cpp:70:7:70:8 | * ... | semmle.label | * ... | +| bdlbb.cpp:75:16:75:23 | call to source | semmle.label | call to source | +| bdlbb.cpp:77:37:77:40 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:77:48:77:51 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:79:37:79:40 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:79:46:79:48 | *src | semmle.label | *src | +| bdlbb.cpp:81:37:81:39 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:81:42:81:44 | *dst | semmle.label | *dst | +| bdlbb.cpp:82:7:82:10 | * ... | semmle.label | * ... | | test.cpp:7:5:7:30 | *ymlStepGenerated_with_body | semmle.label | *ymlStepGenerated_with_body | | test.cpp:7:47:7:52 | value2 | semmle.label | value2 | | test.cpp:7:64:7:69 | value2 | semmle.label | value2 | diff --git a/cpp/ql/test/library-tests/dataflow/external-models/steps.expected b/cpp/ql/test/library-tests/dataflow/external-models/steps.expected index 0fe13460cfbf..9a455ec8ab8c 100644 --- a/cpp/ql/test/library-tests/dataflow/external-models/steps.expected +++ b/cpp/ql/test/library-tests/dataflow/external-models/steps.expected @@ -4,6 +4,17 @@ | azure.cpp:262:5:262:8 | *resp | azure.cpp:262:23:262:28 | ReadToCount output argument | | azure.cpp:287:79:287:98 | call to string | azure.cpp:287:62:287:99 | call to Url | | azure.cpp:289:24:289:56 | call to GetHeader | azure.cpp:289:63:289:65 | call to Value | +| bdlbb.cpp:50:49:50:52 | *call to data | bdlbb.cpp:50:37:50:41 | copy output argument | +| bdlbb.cpp:52:42:52:45 | *blob | bdlbb.cpp:52:37:52:39 | copy output argument | +| bdlbb.cpp:59:49:59:52 | *call to data | bdlbb.cpp:59:37:59:41 | copy output argument | +| bdlbb.cpp:60:18:60:21 | *blob | bdlbb.cpp:60:29:60:32 | *call to buffer | +| bdlbb.cpp:60:29:60:32 | *call to buffer | bdlbb.cpp:60:18:60:38 | *call to data | +| bdlbb.cpp:67:49:67:52 | *call to data | bdlbb.cpp:67:37:67:41 | copy output argument | +| bdlbb.cpp:69:72:69:75 | *blob | bdlbb.cpp:69:12:69:65 | *call to getContiguousRangeOrCopy | +| bdlbb.cpp:69:72:69:75 | *blob | bdlbb.cpp:69:67:69:69 | getContiguousRangeOrCopy output argument | +| bdlbb.cpp:77:48:77:51 | *call to data | bdlbb.cpp:77:37:77:40 | copy output argument | +| bdlbb.cpp:79:46:79:48 | *src | bdlbb.cpp:79:37:79:40 | copy output argument | +| bdlbb.cpp:81:42:81:44 | *dst | bdlbb.cpp:81:37:81:39 | copy output argument | | test.cpp:17:24:17:24 | x | test.cpp:17:10:17:22 | call to ymlStepManual | | test.cpp:21:27:21:27 | x | test.cpp:21:10:21:25 | call to ymlStepGenerated | | test.cpp:25:35:25:35 | x | test.cpp:25:11:25:33 | call to ymlStepManual_with_body | diff --git a/cpp/ql/test/library-tests/dataflow/external-models/validatemodels.expected b/cpp/ql/test/library-tests/dataflow/external-models/validatemodels.expected index 15ae50bddc26..1fbe5da66459 100644 --- a/cpp/ql/test/library-tests/dataflow/external-models/validatemodels.expected +++ b/cpp/ql/test/library-tests/dataflow/external-models/validatemodels.expected @@ -370,6 +370,8 @@ | Dubious signature "(BN_MONT_CTX *,const BIGNUM *,int,const unsigned char *,size_t,uint32_t,uint32_t)" in summary model. | | Dubious signature "(BN_RECP_CTX *,const BIGNUM *,BN_CTX *)" in summary model. | | Dubious signature "(BUF_MEM *,size_t)" in summary model. | +| Dubious signature "(Blob *,int,const Blob &,int,int)" in summary model. | +| Dubious signature "(Blob *,int,const char *,int)" in summary model. | | Dubious signature "(BrotliBitReader *const,uint64_t,uint64_t *)" in summary model. | | Dubious signature "(BrotliDecoderState *,BrotliDecoderStateInternal *,BrotliSharedDictionaryType,size_t,const uint8_t[])" in summary model. | | Dubious signature "(BrotliDecoderState *,BrotliDecoderStateInternal *,brotli_decoder_metadata_start_func,brotli_decoder_metadata_chunk_func,void *)" in summary model. | @@ -2948,6 +2950,7 @@ | Dubious signature "(char *,char *__restrict__,int,FILE *,FILE *__restrict__)" in summary model. | | Dubious signature "(char *,char *__restrict__,size_t,const char *,const char *__restrict__,const tm *,const tm *__restrict__,locale_t)" in summary model. | | Dubious signature "(char *,char,char **)" in summary model. | +| Dubious signature "(char *,const Blob &,int,int)" in summary model. | | Dubious signature "(char *,const char *)" in summary model. | | Dubious signature "(char *,const char **,const char **,const char **,const char **,const char **)" in summary model. | | Dubious signature "(char *,const char *,char **)" in summary model. | From 8116060b90a7cc72bf1dcb1c8b666ca87108508c Mon Sep 17 00:00:00 2001 From: Taus Date: Tue, 1 Sep 2026 13:59:55 +0000 Subject: [PATCH 02/19] Python: Add telemetry for Python analysis version Here' `python_analysis_version` is the version of Python that we are analysing the code as. In practice, all we care about is the major version, but we might as well include the full thing (since it can be overridden on the command line). The `python_runtime_version` is the actual version of Python that ran the extractor. --- .../writing-diagnostics/diagnostics.expected | 20 +++++++ python/extractor/semmle/logging.py | 12 +++++ python/extractor/semmle/util.py | 2 +- python/extractor/semmle/worker.py | 10 +++- python/extractor/tests/test_diagnostics.py | 54 +++++++++++++++++++ 5 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 python/extractor/tests/test_diagnostics.py diff --git a/python/extractor/cli-integration-test/writing-diagnostics/diagnostics.expected b/python/extractor/cli-integration-test/writing-diagnostics/diagnostics.expected index 12a241ad7b68..715fb8dd12f7 100644 --- a/python/extractor/cli-integration-test/writing-diagnostics/diagnostics.expected +++ b/python/extractor/cli-integration-test/writing-diagnostics/diagnostics.expected @@ -161,3 +161,23 @@ "telemetry": true } } +{ + "attributes": { + "extractor_version": "7.1.10", + "python_analysis_version": "3.12", + "python_runtime_version": "3.12.3" + }, + "markdownMessage": "Internal telemetry for the Python extractor.\n\nNo action needed.", + "severity": "note", + "source": { + "extractorName": "python", + "id": "py/extractor/summary", + "name": "Python extractor telemetry" + }, + "timestamp": "2026-09-01T13:41:33.056818Z", + "visibility": { + "cliSummaryTable": false, + "statusPage": false, + "telemetry": true + } +} diff --git a/python/extractor/semmle/logging.py b/python/extractor/semmle/logging.py index 64037163e697..369592534676 100644 --- a/python/extractor/semmle/logging.py +++ b/python/extractor/semmle/logging.py @@ -8,6 +8,9 @@ import multiprocessing import enum import datetime +import platform + +from semmle.util import VERSION, get_analysis_version #Use standard Semmle logging levels @@ -355,6 +358,15 @@ def with_timestamp(self, timestamp): self.timestamp = timestamp return self +def extractor_telemetry_message(): + return (DiagnosticMessage(Source("py/extractor/summary", "Python extractor telemetry"), Severity.NOTE) + .markdown("Internal telemetry for the Python extractor.\n\nNo action needed.") + .attribute("python_analysis_version", get_analysis_version()) + .attribute("python_runtime_version", platform.python_version()) + .attribute("extractor_version", VERSION) + .telemetry() + ) + def get_stack_trace_lines(): """Creates a stack trace for inclusion into the `attributes` part of a diagnostic message. Limits the size of the stack trace to 5000 characters, so as to not make the SARIF file overly big. diff --git a/python/extractor/semmle/util.py b/python/extractor/semmle/util.py index 977d47c69dca..60d215e5bdf4 100644 --- a/python/extractor/semmle/util.py +++ b/python/extractor/semmle/util.py @@ -10,7 +10,7 @@ #Semantic version of extractor. #Update this if any changes are made -VERSION = "7.1.9" +VERSION = "7.1.10" PY_EXTENSIONS = ".py", ".pyw" diff --git a/python/extractor/semmle/worker.py b/python/extractor/semmle/worker.py index ac8231390742..8d771828ada8 100644 --- a/python/extractor/semmle/worker.py +++ b/python/extractor/semmle/worker.py @@ -10,7 +10,7 @@ from semmle.extractors import SuperExtractor, ModulePrinter, SkippedBuiltin from semmle.profiling import get_profiler from semmle.path_rename import renamer_from_options_and_env -from semmle.logging import WARN, recursion_error_message, internal_error_message, Logger +from semmle.logging import WARN, recursion_error_message, internal_error_message, extractor_telemetry_message, Logger from semmle.util import FileExtractable, FolderExtractable class ExtractorFailure(Exception): @@ -239,6 +239,12 @@ def _drain_queue(queue): #Emptied queue as best we can. pass +def _write_extractor_telemetry(diagnostics_writer, logger: Logger): + try: + diagnostics_writer.write(extractor_telemetry_message()) + except OSError as ex: + logger.warning("Failed to write extractor telemetry: %s", ex) + class DiagnosticsWriter(object): def __init__(self, proc_id): self.proc_id = proc_id @@ -276,6 +282,8 @@ def _extract_loop(proc_id, queue, trap_dir, archive, options, reply_queue, logge reply_queue.put(("INTERRUPT", None, None)) sys.exit(2) logger.set_process_id(proc_id) + if write_global_data: + _write_extractor_telemetry(diagnostics_writer, logger) try: if options.trace_only: extractor = ModulePrinter(options, trap_dir, archive, renamer, logger) diff --git a/python/extractor/tests/test_diagnostics.py b/python/extractor/tests/test_diagnostics.py new file mode 100644 index 000000000000..a40f339e3e4b --- /dev/null +++ b/python/extractor/tests/test_diagnostics.py @@ -0,0 +1,54 @@ +import platform + +from semmle import logging +from semmle import util +from semmle import worker + + +def test_extractor_telemetry_message(mocker): + mocker.patch("semmle.logging.get_analysis_version", return_value="3.13") + + message = logging.extractor_telemetry_message().to_dict() + message.pop("timestamp") + + assert message == { + "source": { + "id": "py/extractor/summary", + "name": "Python extractor telemetry", + "extractorName": "python", + }, + "severity": "note", + "markdownMessage": "Internal telemetry for the Python extractor.\n\nNo action needed.", + "visibility": { + "statusPage": False, + "cliSummaryTable": False, + "telemetry": True, + }, + "attributes": { + "python_analysis_version": "3.13", + "python_runtime_version": platform.python_version(), + "extractor_version": util.VERSION, + }, + } + + +def test_write_extractor_telemetry(mocker): + diagnostics_writer = mocker.Mock() + logger = mocker.Mock() + + worker._write_extractor_telemetry(diagnostics_writer, logger) + + diagnostics_writer.write.assert_called_once() + logger.warning.assert_not_called() + + +def test_write_extractor_telemetry_handles_io_error(mocker): + diagnostics_writer = mocker.Mock() + diagnostics_writer.write.side_effect = OSError("write failed") + logger = mocker.Mock() + + worker._write_extractor_telemetry(diagnostics_writer, logger) + + logger.warning.assert_called_once_with( + "Failed to write extractor telemetry: %s", diagnostics_writer.write.side_effect + ) From 9559357bc637f7d5531aeb3152ec472fb61aa3f3 Mon Sep 17 00:00:00 2001 From: AkshayK Date: Thu, 3 Sep 2026 04:21:07 -0400 Subject: [PATCH 03/19] fix(cpp): model bdlbb::BlobBuffer::buffer and trim model comments Address review feedback on the bdlbb::Blob models: - Add a summary for BlobBuffer::buffer(), which returns the bsl::shared_ptr that owns the bytes. No shared_ptr rows are needed: SmartPointer.qll already covers bsl::shared_ptr::get(). - Add a harness case that reads through blob.buffer(0).buffer().get(). - Shorten the section comments in bdlbb.model.yml and the change note. --- .../2026-08-27-bdlbb-blob-models.md | 2 +- cpp/ql/lib/ext/bdlbb.model.yml | 10 +- .../dataflow/external-models/bdlbb.cpp | 15 ++ .../dataflow/external-models/flow.expected | 138 ++++++++++-------- .../dataflow/external-models/steps.expected | 25 ++-- 5 files changed, 111 insertions(+), 79 deletions(-) diff --git a/cpp/ql/lib/change-notes/2026-08-27-bdlbb-blob-models.md b/cpp/ql/lib/change-notes/2026-08-27-bdlbb-blob-models.md index f1db9d3c3116..ed52e5e091e2 100644 --- a/cpp/ql/lib/change-notes/2026-08-27-bdlbb-blob-models.md +++ b/cpp/ql/lib/change-notes/2026-08-27-bdlbb-blob-models.md @@ -1,4 +1,4 @@ --- category: minorAnalysis --- -* Added flow summaries for the BDE `bdlbb::Blob` segmented byte buffer (`BloombergLP::bdlbb`). Taint now flows from a blob to its bytes through the `Blob::buffer`/`BlobBuffer::data` accessor chain and through the `bdlbb::BlobUtil::copy` and `getContiguousRangeOrCopy` helpers, so a blob populated from untrusted input (for example a BlazingMQ message body read via `bmqa::Message::getData`) is tracked into the payload bytes. +* Added flow summaries for the BDE `BloombergLP::bdlbb::Blob` segmented byte buffer. diff --git a/cpp/ql/lib/ext/bdlbb.model.yml b/cpp/ql/lib/ext/bdlbb.model.yml index 135f6c631e83..e5c50207c464 100644 --- a/cpp/ql/lib/ext/bdlbb.model.yml +++ b/cpp/ql/lib/ext/bdlbb.model.yml @@ -5,16 +5,14 @@ extensions: pack: codeql/cpp-all extensible: summaryModel data: # namespace, type, subtypes, name, signature, ext, input, output, kind, provenance - # Accessor chain: a tainted blob taints its buffers, and a tainted buffer taints its bytes. + # Accessor chain - ["BloombergLP::bdlbb", "Blob", true, "buffer", "", "", "Argument[-1]", "ReturnValue[*]", "taint", "manual"] - ["BloombergLP::bdlbb", "BlobBuffer", true, "data", "", "", "Argument[-1]", "ReturnValue[*]", "taint", "manual"] - # BlobUtil read-out: the source blob (Argument[*1]) taints the destination buffer (and the - # returned contiguous range). + - ["BloombergLP::bdlbb", "BlobBuffer", true, "buffer", "", "", "Argument[-1]", "ReturnValue[*]", "taint", "manual"] + # BlobUtil read-out - ["BloombergLP::bdlbb", "BlobUtil", true, "copy", "(char *,const Blob &,int,int)", "", "Argument[*1]", "Argument[*0]", "taint", "manual"] - ["BloombergLP::bdlbb", "BlobUtil", true, "getContiguousRangeOrCopy", "", "", "Argument[*1]", "Argument[*0]", "taint", "manual"] - ["BloombergLP::bdlbb", "BlobUtil", true, "getContiguousRangeOrCopy", "", "", "Argument[*1]", "ReturnValue[*]", "taint", "manual"] - # BlobUtil write-in: the source (Argument[*2]) taints the destination blob. `copy` has two - # write-in overloads, one taking a raw byte buffer and one taking another blob as the source; - # each row pins the exact signature so the int offset/length arguments are never tainted. + # BlobUtil write-in - ["BloombergLP::bdlbb", "BlobUtil", true, "copy", "(Blob *,int,const char *,int)", "", "Argument[*2]", "Argument[*0]", "taint", "manual"] - ["BloombergLP::bdlbb", "BlobUtil", true, "copy", "(Blob *,int,const Blob &,int,int)", "", "Argument[*2]", "Argument[*0]", "taint", "manual"] diff --git a/cpp/ql/test/library-tests/dataflow/external-models/bdlbb.cpp b/cpp/ql/test/library-tests/dataflow/external-models/bdlbb.cpp index a76174c42d46..c8ec9dfd031e 100644 --- a/cpp/ql/test/library-tests/dataflow/external-models/bdlbb.cpp +++ b/cpp/ql/test/library-tests/dataflow/external-models/bdlbb.cpp @@ -13,6 +13,10 @@ namespace bsl { size_t size() const; }; typedef basic_string string; + template class shared_ptr { + public: + T *get() const; + }; } namespace BloombergLP { @@ -20,6 +24,8 @@ namespace bdlbb { class BlobBuffer { public: char *data() const; + bsl::shared_ptr &buffer(); + const bsl::shared_ptr &buffer() const; }; class Blob { @@ -61,6 +67,15 @@ void test_accessor_chain() { sink(*p); // $ ir } +// The get() step comes from the built-in smart pointer model, not from bdlbb.model.yml. +void test_accessor_chain_shared_ptr() { + bsl::string s(source()); + BloombergLP::bdlbb::Blob blob; + BloombergLP::bdlbb::BlobUtil::copy(&blob, 0, s.data(), s.size()); + const char *p = blob.buffer(0).buffer().get(); + sink(*p); // $ ir +} + void test_getContiguousRangeOrCopy() { bsl::string s(source()); BloombergLP::bdlbb::Blob blob; diff --git a/cpp/ql/test/library-tests/dataflow/external-models/flow.expected b/cpp/ql/test/library-tests/dataflow/external-models/flow.expected index 8989ddfa612b..96bd8c372936 100644 --- a/cpp/ql/test/library-tests/dataflow/external-models/flow.expected +++ b/cpp/ql/test/library-tests/dataflow/external-models/flow.expected @@ -96,12 +96,13 @@ models | 95 | Summary: Azure::Core::IO; BodyStream; true; ReadToEnd; ; ; Argument[-1]; ReturnValue.Element; taint; manual | | 96 | Summary: Azure; Nullable; true; Value; ; ; Argument[-1]; ReturnValue[*]; taint; manual | | 97 | Summary: BloombergLP::bdlbb; Blob; true; buffer; ; ; Argument[-1]; ReturnValue[*]; taint; manual | -| 98 | Summary: BloombergLP::bdlbb; BlobBuffer; true; data; ; ; Argument[-1]; ReturnValue[*]; taint; manual | -| 99 | Summary: BloombergLP::bdlbb; BlobUtil; true; copy; (Blob *,int,const Blob &,int,int); ; Argument[*2]; Argument[*0]; taint; manual | -| 100 | Summary: BloombergLP::bdlbb; BlobUtil; true; copy; (Blob *,int,const char *,int); ; Argument[*2]; Argument[*0]; taint; manual | -| 101 | Summary: BloombergLP::bdlbb; BlobUtil; true; copy; (char *,const Blob &,int,int); ; Argument[*1]; Argument[*0]; taint; manual | -| 102 | Summary: BloombergLP::bdlbb; BlobUtil; true; getContiguousRangeOrCopy; ; ; Argument[*1]; ReturnValue[*]; taint; manual | -| 103 | Summary: boost::asio; ; false; buffer; ; ; Argument[*0]; ReturnValue; taint; manual | +| 98 | Summary: BloombergLP::bdlbb; BlobBuffer; true; buffer; ; ; Argument[-1]; ReturnValue[*]; taint; manual | +| 99 | Summary: BloombergLP::bdlbb; BlobBuffer; true; data; ; ; Argument[-1]; ReturnValue[*]; taint; manual | +| 100 | Summary: BloombergLP::bdlbb; BlobUtil; true; copy; (Blob *,int,const Blob &,int,int); ; Argument[*2]; Argument[*0]; taint; manual | +| 101 | Summary: BloombergLP::bdlbb; BlobUtil; true; copy; (Blob *,int,const char *,int); ; Argument[*2]; Argument[*0]; taint; manual | +| 102 | Summary: BloombergLP::bdlbb; BlobUtil; true; copy; (char *,const Blob &,int,int); ; Argument[*1]; Argument[*0]; taint; manual | +| 103 | Summary: BloombergLP::bdlbb; BlobUtil; true; getContiguousRangeOrCopy; ; ; Argument[*1]; ReturnValue[*]; taint; manual | +| 104 | Summary: boost::asio; ; false; buffer; ; ; Argument[*0]; ReturnValue; taint; manual | edges | asio_streams.cpp:87:34:87:44 | read_until output argument | asio_streams.cpp:91:7:91:17 | recv_buffer | provenance | Src:MaD:56 | | asio_streams.cpp:87:34:87:44 | read_until output argument | asio_streams.cpp:93:29:93:39 | recv_buffer | provenance | Src:MaD:56 Sink:MaD:4 | @@ -110,7 +111,7 @@ edges | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | | | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:101:7:101:17 | send_buffer | provenance | | | asio_streams.cpp:100:44:100:62 | call to buffer | asio_streams.cpp:103:29:103:39 | send_buffer | provenance | Sink:MaD:4 | -| asio_streams.cpp:100:64:100:71 | *send_str | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | MaD:103 | +| asio_streams.cpp:100:64:100:71 | *send_str | asio_streams.cpp:100:44:100:62 | call to buffer | provenance | MaD:104 | | azure.cpp:253:48:253:60 | *call to GetBodyStream | azure.cpp:257:5:257:8 | *resp | provenance | | | azure.cpp:253:48:253:60 | *call to GetBodyStream | azure.cpp:262:5:262:8 | *resp | provenance | | | azure.cpp:253:48:253:60 | *call to GetBodyStream | azure.cpp:266:38:266:41 | *resp | provenance | | @@ -150,31 +151,38 @@ edges | azure.cpp:294:38:294:53 | call to operator[] | azure.cpp:295:10:295:20 | contentType | provenance | | | azure.cpp:294:38:294:53 | call to operator[] | azure.cpp:295:10:295:20 | contentType | provenance | | | azure.cpp:295:10:295:20 | contentType | azure.cpp:295:10:295:20 | contentType | provenance | | -| bdlbb.cpp:48:16:48:23 | call to source | bdlbb.cpp:50:49:50:52 | *call to data | provenance | TaintFunction | -| bdlbb.cpp:50:37:50:41 | copy output argument | bdlbb.cpp:52:42:52:45 | *blob | provenance | | -| bdlbb.cpp:50:49:50:52 | *call to data | bdlbb.cpp:50:37:50:41 | copy output argument | provenance | MaD:100 | -| bdlbb.cpp:52:37:52:39 | copy output argument | bdlbb.cpp:53:7:53:10 | * ... | provenance | | -| bdlbb.cpp:52:42:52:45 | *blob | bdlbb.cpp:52:37:52:39 | copy output argument | provenance | MaD:101 | -| bdlbb.cpp:57:16:57:23 | call to source | bdlbb.cpp:59:49:59:52 | *call to data | provenance | TaintFunction | -| bdlbb.cpp:59:37:59:41 | copy output argument | bdlbb.cpp:60:18:60:21 | *blob | provenance | | -| bdlbb.cpp:59:49:59:52 | *call to data | bdlbb.cpp:59:37:59:41 | copy output argument | provenance | MaD:100 | -| bdlbb.cpp:60:18:60:21 | *blob | bdlbb.cpp:60:29:60:32 | *call to buffer | provenance | MaD:97 | -| bdlbb.cpp:60:18:60:38 | *call to data | bdlbb.cpp:60:18:60:38 | *call to data | provenance | | -| bdlbb.cpp:60:18:60:38 | *call to data | bdlbb.cpp:61:7:61:8 | * ... | provenance | | -| bdlbb.cpp:60:29:60:32 | *call to buffer | bdlbb.cpp:60:18:60:38 | *call to data | provenance | MaD:98 | -| bdlbb.cpp:65:16:65:23 | call to source | bdlbb.cpp:67:49:67:52 | *call to data | provenance | TaintFunction | -| bdlbb.cpp:67:37:67:41 | copy output argument | bdlbb.cpp:69:72:69:75 | *blob | provenance | | -| bdlbb.cpp:67:49:67:52 | *call to data | bdlbb.cpp:67:37:67:41 | copy output argument | provenance | MaD:100 | -| bdlbb.cpp:69:12:69:65 | *call to getContiguousRangeOrCopy | bdlbb.cpp:69:12:69:65 | *call to getContiguousRangeOrCopy | provenance | | -| bdlbb.cpp:69:12:69:65 | *call to getContiguousRangeOrCopy | bdlbb.cpp:70:7:70:8 | * ... | provenance | | -| bdlbb.cpp:69:72:69:75 | *blob | bdlbb.cpp:69:12:69:65 | *call to getContiguousRangeOrCopy | provenance | MaD:102 | -| bdlbb.cpp:75:16:75:23 | call to source | bdlbb.cpp:77:48:77:51 | *call to data | provenance | TaintFunction | -| bdlbb.cpp:77:37:77:40 | copy output argument | bdlbb.cpp:79:46:79:48 | *src | provenance | | -| bdlbb.cpp:77:48:77:51 | *call to data | bdlbb.cpp:77:37:77:40 | copy output argument | provenance | MaD:100 | -| bdlbb.cpp:79:37:79:40 | copy output argument | bdlbb.cpp:81:42:81:44 | *dst | provenance | | -| bdlbb.cpp:79:46:79:48 | *src | bdlbb.cpp:79:37:79:40 | copy output argument | provenance | MaD:99 | -| bdlbb.cpp:81:37:81:39 | copy output argument | bdlbb.cpp:82:7:82:10 | * ... | provenance | | -| bdlbb.cpp:81:42:81:44 | *dst | bdlbb.cpp:81:37:81:39 | copy output argument | provenance | MaD:101 | +| bdlbb.cpp:54:16:54:23 | call to source | bdlbb.cpp:56:49:56:52 | *call to data | provenance | TaintFunction | +| bdlbb.cpp:56:37:56:41 | copy output argument | bdlbb.cpp:58:42:58:45 | *blob | provenance | | +| bdlbb.cpp:56:49:56:52 | *call to data | bdlbb.cpp:56:37:56:41 | copy output argument | provenance | MaD:101 | +| bdlbb.cpp:58:37:58:39 | copy output argument | bdlbb.cpp:59:7:59:10 | * ... | provenance | | +| bdlbb.cpp:58:42:58:45 | *blob | bdlbb.cpp:58:37:58:39 | copy output argument | provenance | MaD:102 | +| bdlbb.cpp:63:16:63:23 | call to source | bdlbb.cpp:65:49:65:52 | *call to data | provenance | TaintFunction | +| bdlbb.cpp:65:37:65:41 | copy output argument | bdlbb.cpp:66:18:66:21 | *blob | provenance | | +| bdlbb.cpp:65:49:65:52 | *call to data | bdlbb.cpp:65:37:65:41 | copy output argument | provenance | MaD:101 | +| bdlbb.cpp:66:18:66:21 | *blob | bdlbb.cpp:66:29:66:32 | *call to buffer | provenance | MaD:97 | +| bdlbb.cpp:66:18:66:38 | *call to data | bdlbb.cpp:66:18:66:38 | *call to data | provenance | | +| bdlbb.cpp:66:18:66:38 | *call to data | bdlbb.cpp:67:7:67:8 | * ... | provenance | | +| bdlbb.cpp:66:29:66:32 | *call to buffer | bdlbb.cpp:66:18:66:38 | *call to data | provenance | MaD:99 | +| bdlbb.cpp:72:16:72:23 | call to source | bdlbb.cpp:74:49:74:52 | *call to data | provenance | TaintFunction | +| bdlbb.cpp:74:37:74:41 | copy output argument | bdlbb.cpp:75:18:75:21 | *blob | provenance | | +| bdlbb.cpp:74:49:74:52 | *call to data | bdlbb.cpp:74:37:74:41 | copy output argument | provenance | MaD:101 | +| bdlbb.cpp:75:18:75:21 | *blob | bdlbb.cpp:75:29:75:32 | *call to buffer | provenance | MaD:97 | +| bdlbb.cpp:75:18:75:46 | call to get | bdlbb.cpp:76:7:76:8 | * ... | provenance | | +| bdlbb.cpp:75:29:75:32 | *call to buffer | bdlbb.cpp:75:39:75:41 | *call to buffer | provenance | MaD:98 | +| bdlbb.cpp:75:39:75:41 | *call to buffer | bdlbb.cpp:75:18:75:46 | call to get | provenance | DataFlowFunction | +| bdlbb.cpp:80:16:80:23 | call to source | bdlbb.cpp:82:49:82:52 | *call to data | provenance | TaintFunction | +| bdlbb.cpp:82:37:82:41 | copy output argument | bdlbb.cpp:84:72:84:75 | *blob | provenance | | +| bdlbb.cpp:82:49:82:52 | *call to data | bdlbb.cpp:82:37:82:41 | copy output argument | provenance | MaD:101 | +| bdlbb.cpp:84:12:84:65 | *call to getContiguousRangeOrCopy | bdlbb.cpp:84:12:84:65 | *call to getContiguousRangeOrCopy | provenance | | +| bdlbb.cpp:84:12:84:65 | *call to getContiguousRangeOrCopy | bdlbb.cpp:85:7:85:8 | * ... | provenance | | +| bdlbb.cpp:84:72:84:75 | *blob | bdlbb.cpp:84:12:84:65 | *call to getContiguousRangeOrCopy | provenance | MaD:103 | +| bdlbb.cpp:90:16:90:23 | call to source | bdlbb.cpp:92:48:92:51 | *call to data | provenance | TaintFunction | +| bdlbb.cpp:92:37:92:40 | copy output argument | bdlbb.cpp:94:46:94:48 | *src | provenance | | +| bdlbb.cpp:92:48:92:51 | *call to data | bdlbb.cpp:92:37:92:40 | copy output argument | provenance | MaD:101 | +| bdlbb.cpp:94:37:94:40 | copy output argument | bdlbb.cpp:96:42:96:44 | *dst | provenance | | +| bdlbb.cpp:94:46:94:48 | *src | bdlbb.cpp:94:37:94:40 | copy output argument | provenance | MaD:100 | +| bdlbb.cpp:96:37:96:39 | copy output argument | bdlbb.cpp:97:7:97:10 | * ... | provenance | | +| bdlbb.cpp:96:42:96:44 | *dst | bdlbb.cpp:96:37:96:39 | copy output argument | provenance | MaD:102 | | test.cpp:7:47:7:52 | value2 | test.cpp:7:64:7:69 | value2 | provenance | | | test.cpp:7:64:7:69 | value2 | test.cpp:7:5:7:30 | *ymlStepGenerated_with_body | provenance | | | test.cpp:10:10:10:18 | call to ymlSource | test.cpp:10:10:10:18 | call to ymlSource | provenance | Src:MaD:48 | @@ -563,35 +571,43 @@ nodes | azure.cpp:295:10:295:20 | contentType | semmle.label | contentType | | azure.cpp:295:10:295:20 | contentType | semmle.label | contentType | | azure.cpp:295:10:295:20 | contentType | semmle.label | contentType | -| bdlbb.cpp:48:16:48:23 | call to source | semmle.label | call to source | -| bdlbb.cpp:50:37:50:41 | copy output argument | semmle.label | copy output argument | -| bdlbb.cpp:50:49:50:52 | *call to data | semmle.label | *call to data | -| bdlbb.cpp:52:37:52:39 | copy output argument | semmle.label | copy output argument | -| bdlbb.cpp:52:42:52:45 | *blob | semmle.label | *blob | -| bdlbb.cpp:53:7:53:10 | * ... | semmle.label | * ... | -| bdlbb.cpp:57:16:57:23 | call to source | semmle.label | call to source | -| bdlbb.cpp:59:37:59:41 | copy output argument | semmle.label | copy output argument | -| bdlbb.cpp:59:49:59:52 | *call to data | semmle.label | *call to data | -| bdlbb.cpp:60:18:60:21 | *blob | semmle.label | *blob | -| bdlbb.cpp:60:18:60:38 | *call to data | semmle.label | *call to data | -| bdlbb.cpp:60:18:60:38 | *call to data | semmle.label | *call to data | -| bdlbb.cpp:60:29:60:32 | *call to buffer | semmle.label | *call to buffer | -| bdlbb.cpp:61:7:61:8 | * ... | semmle.label | * ... | -| bdlbb.cpp:65:16:65:23 | call to source | semmle.label | call to source | -| bdlbb.cpp:67:37:67:41 | copy output argument | semmle.label | copy output argument | -| bdlbb.cpp:67:49:67:52 | *call to data | semmle.label | *call to data | -| bdlbb.cpp:69:12:69:65 | *call to getContiguousRangeOrCopy | semmle.label | *call to getContiguousRangeOrCopy | -| bdlbb.cpp:69:12:69:65 | *call to getContiguousRangeOrCopy | semmle.label | *call to getContiguousRangeOrCopy | -| bdlbb.cpp:69:72:69:75 | *blob | semmle.label | *blob | -| bdlbb.cpp:70:7:70:8 | * ... | semmle.label | * ... | -| bdlbb.cpp:75:16:75:23 | call to source | semmle.label | call to source | -| bdlbb.cpp:77:37:77:40 | copy output argument | semmle.label | copy output argument | -| bdlbb.cpp:77:48:77:51 | *call to data | semmle.label | *call to data | -| bdlbb.cpp:79:37:79:40 | copy output argument | semmle.label | copy output argument | -| bdlbb.cpp:79:46:79:48 | *src | semmle.label | *src | -| bdlbb.cpp:81:37:81:39 | copy output argument | semmle.label | copy output argument | -| bdlbb.cpp:81:42:81:44 | *dst | semmle.label | *dst | -| bdlbb.cpp:82:7:82:10 | * ... | semmle.label | * ... | +| bdlbb.cpp:54:16:54:23 | call to source | semmle.label | call to source | +| bdlbb.cpp:56:37:56:41 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:56:49:56:52 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:58:37:58:39 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:58:42:58:45 | *blob | semmle.label | *blob | +| bdlbb.cpp:59:7:59:10 | * ... | semmle.label | * ... | +| bdlbb.cpp:63:16:63:23 | call to source | semmle.label | call to source | +| bdlbb.cpp:65:37:65:41 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:65:49:65:52 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:66:18:66:21 | *blob | semmle.label | *blob | +| bdlbb.cpp:66:18:66:38 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:66:18:66:38 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:66:29:66:32 | *call to buffer | semmle.label | *call to buffer | +| bdlbb.cpp:67:7:67:8 | * ... | semmle.label | * ... | +| bdlbb.cpp:72:16:72:23 | call to source | semmle.label | call to source | +| bdlbb.cpp:74:37:74:41 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:74:49:74:52 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:75:18:75:21 | *blob | semmle.label | *blob | +| bdlbb.cpp:75:18:75:46 | call to get | semmle.label | call to get | +| bdlbb.cpp:75:29:75:32 | *call to buffer | semmle.label | *call to buffer | +| bdlbb.cpp:75:39:75:41 | *call to buffer | semmle.label | *call to buffer | +| bdlbb.cpp:76:7:76:8 | * ... | semmle.label | * ... | +| bdlbb.cpp:80:16:80:23 | call to source | semmle.label | call to source | +| bdlbb.cpp:82:37:82:41 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:82:49:82:52 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:84:12:84:65 | *call to getContiguousRangeOrCopy | semmle.label | *call to getContiguousRangeOrCopy | +| bdlbb.cpp:84:12:84:65 | *call to getContiguousRangeOrCopy | semmle.label | *call to getContiguousRangeOrCopy | +| bdlbb.cpp:84:72:84:75 | *blob | semmle.label | *blob | +| bdlbb.cpp:85:7:85:8 | * ... | semmle.label | * ... | +| bdlbb.cpp:90:16:90:23 | call to source | semmle.label | call to source | +| bdlbb.cpp:92:37:92:40 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:92:48:92:51 | *call to data | semmle.label | *call to data | +| bdlbb.cpp:94:37:94:40 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:94:46:94:48 | *src | semmle.label | *src | +| bdlbb.cpp:96:37:96:39 | copy output argument | semmle.label | copy output argument | +| bdlbb.cpp:96:42:96:44 | *dst | semmle.label | *dst | +| bdlbb.cpp:97:7:97:10 | * ... | semmle.label | * ... | | test.cpp:7:5:7:30 | *ymlStepGenerated_with_body | semmle.label | *ymlStepGenerated_with_body | | test.cpp:7:47:7:52 | value2 | semmle.label | value2 | | test.cpp:7:64:7:69 | value2 | semmle.label | value2 | diff --git a/cpp/ql/test/library-tests/dataflow/external-models/steps.expected b/cpp/ql/test/library-tests/dataflow/external-models/steps.expected index 9a455ec8ab8c..42d2c0183c34 100644 --- a/cpp/ql/test/library-tests/dataflow/external-models/steps.expected +++ b/cpp/ql/test/library-tests/dataflow/external-models/steps.expected @@ -4,17 +4,20 @@ | azure.cpp:262:5:262:8 | *resp | azure.cpp:262:23:262:28 | ReadToCount output argument | | azure.cpp:287:79:287:98 | call to string | azure.cpp:287:62:287:99 | call to Url | | azure.cpp:289:24:289:56 | call to GetHeader | azure.cpp:289:63:289:65 | call to Value | -| bdlbb.cpp:50:49:50:52 | *call to data | bdlbb.cpp:50:37:50:41 | copy output argument | -| bdlbb.cpp:52:42:52:45 | *blob | bdlbb.cpp:52:37:52:39 | copy output argument | -| bdlbb.cpp:59:49:59:52 | *call to data | bdlbb.cpp:59:37:59:41 | copy output argument | -| bdlbb.cpp:60:18:60:21 | *blob | bdlbb.cpp:60:29:60:32 | *call to buffer | -| bdlbb.cpp:60:29:60:32 | *call to buffer | bdlbb.cpp:60:18:60:38 | *call to data | -| bdlbb.cpp:67:49:67:52 | *call to data | bdlbb.cpp:67:37:67:41 | copy output argument | -| bdlbb.cpp:69:72:69:75 | *blob | bdlbb.cpp:69:12:69:65 | *call to getContiguousRangeOrCopy | -| bdlbb.cpp:69:72:69:75 | *blob | bdlbb.cpp:69:67:69:69 | getContiguousRangeOrCopy output argument | -| bdlbb.cpp:77:48:77:51 | *call to data | bdlbb.cpp:77:37:77:40 | copy output argument | -| bdlbb.cpp:79:46:79:48 | *src | bdlbb.cpp:79:37:79:40 | copy output argument | -| bdlbb.cpp:81:42:81:44 | *dst | bdlbb.cpp:81:37:81:39 | copy output argument | +| bdlbb.cpp:56:49:56:52 | *call to data | bdlbb.cpp:56:37:56:41 | copy output argument | +| bdlbb.cpp:58:42:58:45 | *blob | bdlbb.cpp:58:37:58:39 | copy output argument | +| bdlbb.cpp:65:49:65:52 | *call to data | bdlbb.cpp:65:37:65:41 | copy output argument | +| bdlbb.cpp:66:18:66:21 | *blob | bdlbb.cpp:66:29:66:32 | *call to buffer | +| bdlbb.cpp:66:29:66:32 | *call to buffer | bdlbb.cpp:66:18:66:38 | *call to data | +| bdlbb.cpp:74:49:74:52 | *call to data | bdlbb.cpp:74:37:74:41 | copy output argument | +| bdlbb.cpp:75:18:75:21 | *blob | bdlbb.cpp:75:29:75:32 | *call to buffer | +| bdlbb.cpp:75:29:75:32 | *call to buffer | bdlbb.cpp:75:39:75:41 | *call to buffer | +| bdlbb.cpp:82:49:82:52 | *call to data | bdlbb.cpp:82:37:82:41 | copy output argument | +| bdlbb.cpp:84:72:84:75 | *blob | bdlbb.cpp:84:12:84:65 | *call to getContiguousRangeOrCopy | +| bdlbb.cpp:84:72:84:75 | *blob | bdlbb.cpp:84:67:84:69 | getContiguousRangeOrCopy output argument | +| bdlbb.cpp:92:48:92:51 | *call to data | bdlbb.cpp:92:37:92:40 | copy output argument | +| bdlbb.cpp:94:46:94:48 | *src | bdlbb.cpp:94:37:94:40 | copy output argument | +| bdlbb.cpp:96:42:96:44 | *dst | bdlbb.cpp:96:37:96:39 | copy output argument | | test.cpp:17:24:17:24 | x | test.cpp:17:10:17:22 | call to ymlStepManual | | test.cpp:21:27:21:27 | x | test.cpp:21:10:21:25 | call to ymlStepGenerated | | test.cpp:25:35:25:35 | x | test.cpp:25:11:25:33 | call to ymlStepManual_with_body | From 703d68037c603445e1f033561acf801d6dd1925c Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 3 Sep 2026 15:29:55 +0200 Subject: [PATCH 04/19] Java: make JDK 11 version normalisation in gradle buildless test robust `java.version` may carry a fourth `$PATCH` component (JEP 322), as in Temurin `jdk-11.0.32.1+1`. The previous pattern matched exactly three components, so the trailing `.1` survived and the test reported `11.1`. Accept any number of version components, and anchor on the surrounding quotes so the substitution only rewrites a whole JSON string rather than version-like text elsewhere in the diagnostics. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../gradle-sample-without-wrapper-or-gradle-buildless/test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/java/ql/integration-tests/java/gradle-sample-without-wrapper-or-gradle-buildless/test.py b/java/ql/integration-tests/java/gradle-sample-without-wrapper-or-gradle-buildless/test.py index 3aaee01f4055..a312878e107f 100644 --- a/java/ql/integration-tests/java/gradle-sample-without-wrapper-or-gradle-buildless/test.py +++ b/java/ql/integration-tests/java/gradle-sample-without-wrapper-or-gradle-buildless/test.py @@ -6,7 +6,9 @@ # The version of gradle used doesn't work on java 17 def test(codeql, use_java_11, java, environment, check_diagnostics): check_diagnostics.redact += ["attributes.java_vendor"] - check_diagnostics.replacements = [("11\\.[0-9]+\\.[0-9]+", "11")] + # the JDK build provided by the CI runner image may report any number of version components + # (e.g. `11.0.32` or `11.0.32.1`), so keep only the feature version + check_diagnostics.replacements = [(r'"11(\.[0-9]+)+"', '"11"')] gradle_override_dir = pathlib.Path(tempfile.mkdtemp()) if runs_on.windows: (gradle_override_dir / "gradle.bat").write_text("@echo off\nexit /b 2\n") From 7aa084c9fd5f080f7b45a50e6decc778b5951133 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Thu, 3 Sep 2026 18:55:10 +0100 Subject: [PATCH 05/19] C++: Fix join in AliasedSSA. Before (on an internal Microsoft repo): Evaluated relational algebra for predicate AliasedSSA::AllocationMemoryLocation.getVirtualVariable/0#dispred#8debd926@f62ba08v with tuple counts: 5606472 ~0% {2} r1 = AliasedSSA::AllocationMemoryLocation#57439a9b_10#join_rhs AND NOT `AliasAnalysis::allocationEscapes/1#93dc9772`(FIRST 1) 5534569 ~0% {2} r2 = r1 AND NOT `project#AliasedSSA::getGroupedMemoryLocation/3#14ef79fc#ffbf`(FIRST 1) 10613026097 ~0% {2} | JOIN WITH AliasedSSA::AllocationMemoryLocation#57439a9b_10#join_rhs ON FIRST 1 OUTPUT Rhs.1, Lhs.1 5534569 ~6% {2} | JOIN WITH AliasedSSA::VirtualVariable#5712df39 ON FIRST 1 OUTPUT Lhs.1, Lhs.0 759836 ~0% {2} r3 = JOIN AliasedSSA::AllocationMemoryLocation#57439a9b_10#join_rhs WITH `AliasAnalysis::allocationEscapes/1#93dc9772` ON FIRST 1 OUTPUT Lhs.0, Lhs.1 759832 ~6% {3} | JOIN WITH `AliasConfiguration::Allocation.getEnclosingIRFunction/0#dispred#3254a7ee` ON FIRST 1 OUTPUT Rhs.1, _, Lhs.1 759832 ~0% {3} | REWRITE WITH Out.1 := false 759832 ~1% {2} | JOIN WITH num#AliasedSSA::TAllAliasedMemory#4bb632db ON FIRST 2 OUTPUT Rhs.2, Lhs.2 759832 ~1% {2} | JOIN WITH AliasedSSA::VirtualVariable#5712df39 ON FIRST 1 OUTPUT Lhs.1, Lhs.0 5606472 ~0% {4} r4 = SCAN r1 OUTPUT In.0, _, _, In.1 5606472 ~0% {4} | REWRITE WITH Out.1 := false, Out.2 := false 71903 ~5% {2} | JOIN WITH `AliasedSSA::getGroupedMemoryLocation/3#14ef79fc#ffbf` ON FIRST 3 OUTPUT Rhs.3, Lhs.3 71903 ~3% {2} | JOIN WITH `AliasedSSA::GroupedMemoryLocation.getVirtualVariable/0#dispred#143f2d1b` ON FIRST 1 OUTPUT Rhs.1, Lhs.1 71903 ~2% {2} | JOIN WITH AliasedSSA::VirtualVariable#5712df39 ON FIRST 1 OUTPUT Lhs.1, Lhs.0 6366304 ~5% {2} r5 = r2 UNION r3 UNION r4 return r5 After: [2026-09-03 14:39:34] Evaluated non-recursive predicate AliasedSSA::getAllocationMemoryLocation/1#171552fa@42e30agr in 15ms (size: 505315). Evaluated relational algebra for predicate AliasedSSA::getAllocationMemoryLocation/1#171552fa@42e30agr with tuple counts: 505315 ~2% {2} r1 = JOIN AliasedSSA::VirtualVariable#5712df39 WITH `AliasedSSA::MemoryLocation0.getAnAllocation/0#dispred#f0047858` ON FIRST 1 OUTPUT Rhs.1, Lhs.0 return r1 [2026-09-03 14:39:39] Evaluated non-recursive predicate AliasedSSA::AllocationMemoryLocation.getVirtualVariable/0#dispred#8debd926@21855asb in 1811ms (size: 6366304). Evaluated relational algebra for predicate AliasedSSA::AllocationMemoryLocation.getVirtualVariable/0#dispred#8debd926@21855asb with tuple counts: 5606472 ~0% {2} r1 = AliasedSSA::AllocationMemoryLocation#57439a9b_10#join_rhs AND NOT `AliasAnalysis::allocationEscapes/1#93dc9772`(FIRST 1) 5534569 ~0% {2} r2 = r1 AND NOT `project#AliasedSSA::getGroupedMemoryLocation/3#14ef79fc#ffbf`(FIRST 1) 5534569 ~1% {2} | JOIN WITH `AliasedSSA::getAllocationMemoryLocation/1#171552fa` ON FIRST 1 OUTPUT Rhs.1, Lhs.1 5534569 ~6% {2} | JOIN WITH AliasedSSA::VirtualVariable#5712df39 ON FIRST 1 OUTPUT Lhs.1, Lhs.0 759836 ~0% {2} r3 = JOIN AliasedSSA::AllocationMemoryLocation#57439a9b_10#join_rhs WITH `AliasAnalysis::allocationEscapes/1#93dc9772` ON FIRST 1 OUTPUT Lhs.0, Lhs.1 759832 ~6% {3} | JOIN WITH `AliasConfiguration::Allocation.getEnclosingIRFunction/0#dispred#3254a7ee` ON FIRST 1 OUTPUT Rhs.1, _, Lhs.1 759832 ~0% {3} | REWRITE WITH Out.1 := false 759832 ~1% {2} | JOIN WITH num#AliasedSSA::TAllAliasedMemory#4bb632db ON FIRST 2 OUTPUT Rhs.2, Lhs.2 759832 ~1% {2} | JOIN WITH AliasedSSA::VirtualVariable#5712df39 ON FIRST 1 OUTPUT Lhs.1, Lhs.0 5606472 ~0% {4} r4 = SCAN r1 OUTPUT In.0, _, _, In.1 5606472 ~0% {4} | REWRITE WITH Out.1 := false, Out.2 := false 71903 ~5% {2} | JOIN WITH `AliasedSSA::getGroupedMemoryLocation/3#14ef79fc#ffbf` ON FIRST 3 OUTPUT Rhs.3, Lhs.3 71903 ~3% {2} | JOIN WITH `AliasedSSA::GroupedMemoryLocation.getVirtualVariable/0#dispred#143f2d1b` ON FIRST 1 OUTPUT Rhs.1, Lhs.1 71903 ~2% {2} | JOIN WITH AliasedSSA::VirtualVariable#5712df39 ON FIRST 1 OUTPUT Lhs.1, Lhs.0 6366304 ~5% {2} r5 = r2 UNION r3 UNION r4 return r5 --- .../ir/implementation/aliased_ssa/internal/AliasedSSA.qll | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/AliasedSSA.qll b/cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/AliasedSSA.qll index 2ace50221313..59ee08973e1a 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/AliasedSSA.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/implementation/aliased_ssa/internal/AliasedSSA.qll @@ -295,6 +295,11 @@ abstract class MemoryLocation0 extends TMemoryLocation { */ abstract class VirtualVariable extends MemoryLocation0 { } +pragma[nomagic] +private VirtualVariable getAllocationMemoryLocation(Allocation alloc) { + result.getAnAllocation() = alloc +} + abstract class AllocationMemoryLocation extends MemoryLocation0 { Allocation var; boolean isMayAccess; @@ -313,7 +318,7 @@ abstract class AllocationMemoryLocation extends MemoryLocation0 { result = getGroupedMemoryLocation(var, false, false).getVirtualVariable() or not exists(getGroupedMemoryLocation(var, false, false)) and - result.(AllocationMemoryLocation).getAnAllocation() = var + result = getAllocationMemoryLocation(var) ) } From c99bff955ecdd663e298955a4df8128c4861af79 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Thu, 3 Sep 2026 18:57:10 +0100 Subject: [PATCH 06/19] C++: Fix join in IR construction. Before (on an internal Microsoft repo): [2026-08-28 14:36:46] Evaluated non-recursive predicate TranslatedElement::TranslatedElement.getInstructionSuccessorInternal/2#dispred#a6e054ca@2248725r in 470505ms (size: 136995987). Evaluated relational algebra for predicate TranslatedElement::TranslatedElement.getInstructionSuccessorInternal/2#dispred#a6e054ca@2248725r with tuple counts: 437 ~0% {2} r158 = JOIN EdgeKind::EdgeKindImpl#8ed21aeb WITH num#InstructionTag::CallTargetTag#8c4ab419 CARTESIAN PRODUCT OUTPUT Lhs.0, Rhs.0 437 ~0% {3} | JOIN WITH num#InstructionTag::CallTag#a77d4021 CARTESIAN PRODUCT OUTPUT Rhs.0, Lhs.0, Lhs.1 1070556919 ~0% {4} | JOIN WITH `TranslatedElement::TranslatedElement.getInstruction/1#dispred#f6df9482_102#join_rhs` ON FIRST 1 OUTPUT Rhs.1, Lhs.1, Lhs.2, Rhs.2 316388 ~1% {4} | JOIN WITH TranslatedInitialization::TranslatedDefaultFieldInitialization#d9c761ed ON FIRST 1 OUTPUT Lhs.0, Lhs.2, Lhs.1, Lhs.3 After: [2026-09-03 13:56:28] Evaluated non-recursive predicate TranslatedInitialization::getCallInstruction/1#c88d849b@3e960544 in 40ms (size: 724). Evaluated relational algebra for predicate TranslatedInitialization::getCallInstruction/1#c88d849b@3e960544 with tuple counts: 724 ~3% {2} r1 = JOIN `_TranslatedElement::TranslatedElement.getInstruction/1#dispred#f6df9482_102#join_rhs_num#Instruction__#shared#1` WITH TranslatedInitialization::TranslatedDefaultFieldInitialization#d9c761ed ON FIRST 1 OUTPUT Lhs.0, Lhs.1 return r1 437 ~0% {2} r5 = JOIN EdgeKind::EdgeKindImpl#8ed21aeb WITH num#InstructionTag::CallTargetTag#8c4ab419 CARTESIAN PRODUCT OUTPUT Lhs.0, Rhs.0 316388 ~1% {4} | JOIN WITH `TranslatedInitialization::getCallInstruction/1#c88d849b` CARTESIAN PRODUCT OUTPUT Rhs.0, Lhs.1, Lhs.0, Rhs.1 --- .../raw/internal/TranslatedInitialization.qll | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedInitialization.qll b/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedInitialization.qll index 10c033131225..c24cb98d2bd9 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedInitialization.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/implementation/raw/internal/TranslatedInitialization.qll @@ -618,6 +618,11 @@ class TranslatedExplicitFieldInitialization extends TranslatedNonDefaultFieldIni override int getPosition() { result = position } } +pragma[nomagic] +private Instruction getCallInstruction(TranslatedDefaultFieldInitialization tdfi) { + result = tdfi.getInstruction(CallTag()) +} + /** * The IR translation of the initialization of a field from an element of an initializer * list where default initialization is used. @@ -642,7 +647,7 @@ class TranslatedDefaultFieldInitialization extends TranslatedFieldInitialization override Instruction getInstructionSuccessorInternal(InstructionTag tag, EdgeKind kind) { tag = CallTargetTag() and - result = this.getInstruction(CallTag()) + result = getCallInstruction(this) or tag = CallTag() and result = this.getSideEffects().getFirstInstruction(kind) From 3dcfd08f929f259ae6de5c88154047fc43ad6aa1 Mon Sep 17 00:00:00 2001 From: Taus Date: Wed, 2 Sep 2026 11:52:49 +0000 Subject: [PATCH 07/19] Python: Add telemetry for parser usage Adds statistics on how many files were extracted using the old parser and using the tree-sitter parser. Because parsing is done in parallel across many workers, I opted not to consolidate these statistics for the entire run. Instead, we emit the statistics for each worker and then need to aggregate themselves after the telemetry has been ingested. (In practice the number of workers is ~16 at most, so is unlikely to be an issue.) In terms of implementation, I opted to simply extend the existing `DiagnosticsWriter` object (instantiatied once per worker) with methods for counting the number of parsed files, and then thread this object through to `modules.py` where the magic happens. Finally, this also required instantiating such an object in cases where we call directly into the extractor for debugging purposes (e.g. dumping the AST or CFG). Note that in these cases we do not actually print any diagnostics, so it's harmless to create these objects. As for tests, we add a new separate CLI integration test that checks the behaviour against a database that contains two files -- one that can be parsed with the old parser and one that requires the new one. The existing diagnostics test is modified slightly so that it ignores these statistics (as we cannot guarantee their exact form due to worker nondeterminism). --- .../parser-telemetry/repo_dir/old_parser.py | 1 + .../repo_dir/tree_sitter_parser.py | 3 + .../parser-telemetry/test.sh | 17 +++ .../parser-telemetry/test_parser_telemetry.py | 25 +++++ .../test_diagnostics_output.py | 9 +- .../semmle/extractors/module_printer.py | 4 +- .../semmle/extractors/py_extractor.py | 3 +- python/extractor/semmle/logging.py | 8 ++ python/extractor/semmle/python/finder.py | 4 +- python/extractor/semmle/python/modules.py | 5 +- .../semmle/python/parser/dump_ast.py | 3 +- python/extractor/semmle/python/passes/flow.py | 3 +- python/extractor/semmle/worker.py | 25 ++++- python/extractor/tests/test_diagnostics.py | 105 ++++++++++++++++++ 14 files changed, 205 insertions(+), 10 deletions(-) create mode 100644 python/extractor/cli-integration-test/parser-telemetry/repo_dir/old_parser.py create mode 100644 python/extractor/cli-integration-test/parser-telemetry/repo_dir/tree_sitter_parser.py create mode 100755 python/extractor/cli-integration-test/parser-telemetry/test.sh create mode 100644 python/extractor/cli-integration-test/parser-telemetry/test_parser_telemetry.py diff --git a/python/extractor/cli-integration-test/parser-telemetry/repo_dir/old_parser.py b/python/extractor/cli-integration-test/parser-telemetry/repo_dir/old_parser.py new file mode 100644 index 000000000000..7d4290a117a4 --- /dev/null +++ b/python/extractor/cli-integration-test/parser-telemetry/repo_dir/old_parser.py @@ -0,0 +1 @@ +x = 1 diff --git a/python/extractor/cli-integration-test/parser-telemetry/repo_dir/tree_sitter_parser.py b/python/extractor/cli-integration-test/parser-telemetry/repo_dir/tree_sitter_parser.py new file mode 100644 index 000000000000..08a86e1f4c6d --- /dev/null +++ b/python/extractor/cli-integration-test/parser-telemetry/repo_dir/tree_sitter_parser.py @@ -0,0 +1,3 @@ +match 1: + case 1: + pass diff --git a/python/extractor/cli-integration-test/parser-telemetry/test.sh b/python/extractor/cli-integration-test/parser-telemetry/test.sh new file mode 100755 index 000000000000..86219c86bf12 --- /dev/null +++ b/python/extractor/cli-integration-test/parser-telemetry/test.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -Eeuo pipefail # see https://vaneyckt.io/posts/safer_bash_scripts_with_set_euxo_pipefail/ + +set -x + +CODEQL=${CODEQL:-codeql} + +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +cd "$SCRIPTDIR" + +rm -rf db + +$CODEQL database create db --language python --source-root repo_dir/ +python3 test_parser_telemetry.py db + +rm -rf db diff --git a/python/extractor/cli-integration-test/parser-telemetry/test_parser_telemetry.py b/python/extractor/cli-integration-test/parser-telemetry/test_parser_telemetry.py new file mode 100644 index 000000000000..c848f63eacd5 --- /dev/null +++ b/python/extractor/cli-integration-test/parser-telemetry/test_parser_telemetry.py @@ -0,0 +1,25 @@ +import glob +import json +import os +import sys + + +database = sys.argv[1] +diagnostics = [] +diagnostic_dir = os.path.join(database, "diagnostic", "extractors", "python") +for path in glob.glob(os.path.join(diagnostic_dir, "*.jsonl")): + with open(path) as diagnostic_file: + diagnostics.extend(json.loads(line) for line in diagnostic_file) +parser_statistics = [ + diagnostic + for diagnostic in diagnostics + if diagnostic["source"]["id"] == "py/extractor/parser-statistics" +] +actual = ( + sum(diagnostic["attributes"]["old_parser_file_count"] for diagnostic in parser_statistics), + sum( + diagnostic["attributes"]["tree_sitter_parser_file_count"] + for diagnostic in parser_statistics + ), +) +assert actual == (1, 1), actual diff --git a/python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py b/python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py index 0dce022a0f95..fae7bc93d88b 100644 --- a/python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py +++ b/python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py @@ -4,4 +4,11 @@ import diagnostics_test_utils test_db = "db" -diagnostics_test_utils.check_diagnostics(".", test_db, skip_attributes=True) +diagnostics_test_utils.check_diagnostics( + ".", + test_db, + skip_attributes=True, + replacements={ + r'"py/extractor/parser-statistics"': '"cli/py/extractor/parser-statistics"' + }, +) diff --git a/python/extractor/semmle/extractors/module_printer.py b/python/extractor/semmle/extractors/module_printer.py index d2f4a6cc92bd..aeaa8e63f5eb 100644 --- a/python/extractor/semmle/extractors/module_printer.py +++ b/python/extractor/semmle/extractors/module_printer.py @@ -6,9 +6,9 @@ class ModulePrinter(object): name = "module printer" - def __init__(self, options, trap_folder, src_archive, renamer, logger): + def __init__(self, options, trap_folder, src_archive, renamer, logger, diagnostics_writer): self.logger = logger - self.py_extractor = PythonExtractor(options, trap_folder, src_archive, logger) + self.py_extractor = PythonExtractor(options, trap_folder, src_archive, logger, diagnostics_writer) def process(self, unit): imports = () diff --git a/python/extractor/semmle/extractors/py_extractor.py b/python/extractor/semmle/extractors/py_extractor.py index 8014063b3cb8..3b5a37fb7a96 100644 --- a/python/extractor/semmle/extractors/py_extractor.py +++ b/python/extractor/semmle/extractors/py_extractor.py @@ -16,6 +16,7 @@ def __init__(self, options, trap_folder, src_archive, logger: Logger, diagnostic self.module_extractor = extractor.Extractor.from_options(options, trap_folder, src_archive, logger, diagnostics_writer) self.finder = finder.Finder.from_options_and_env(options, logger) self.importer = imports.importer_from_options(options, self.finder, logger) + self.diagnostics_writer = diagnostics_writer def _get_module_and_imports(self, unit): if not isinstance(unit, util.FileExtractable): @@ -24,7 +25,7 @@ def _get_module_and_imports(self, unit): module = self.finder.from_extractable(unit) if module is None: return None, () - py_module = module.load(self.logger) + py_module = module.load(self.logger, self.diagnostics_writer) if py_module is None: return None, () imports = set(mod.get_extractable() for mod in self.importer.get_imports(module, py_module)) diff --git a/python/extractor/semmle/logging.py b/python/extractor/semmle/logging.py index 369592534676..6f5255a7a9d5 100644 --- a/python/extractor/semmle/logging.py +++ b/python/extractor/semmle/logging.py @@ -367,6 +367,14 @@ def extractor_telemetry_message(): .telemetry() ) +def parser_statistics_telemetry_message(old_parser_file_count, tree_sitter_parser_file_count): + return (DiagnosticMessage(Source("py/extractor/parser-statistics", "Python parser statistics"), Severity.NOTE) + .markdown("Internal parser telemetry for the Python extractor.\n\nNo action needed.") + .attribute("old_parser_file_count", old_parser_file_count) + .attribute("tree_sitter_parser_file_count", tree_sitter_parser_file_count) + .telemetry() + ) + def get_stack_trace_lines(): """Creates a stack trace for inclusion into the `attributes` part of a diagnostic message. Limits the size of the stack trace to 5000 characters, so as to not make the SARIF file overly big. diff --git a/python/extractor/semmle/python/finder.py b/python/extractor/semmle/python/finder.py index 632ef920d055..46e63f5e578f 100644 --- a/python/extractor/semmle/python/finder.py +++ b/python/extractor/semmle/python/finder.py @@ -65,8 +65,8 @@ def all_sub_modules(self): def get_extractable(self): return FileExtractable(self.path) - def load(self, logger=None): - return PythonSourceModule(self.name, self.path, logger=logger) + def load(self, logger, diagnostics_writer): + return PythonSourceModule(self.name, self.path, logger=logger, diagnostics_writer=diagnostics_writer) def __str__(self): return "Python module at %s" % self.path diff --git a/python/extractor/semmle/python/modules.py b/python/extractor/semmle/python/modules.py index 810c4e060f7b..7192f97cfb11 100644 --- a/python/extractor/semmle/python/modules.py +++ b/python/extractor/semmle/python/modules.py @@ -18,7 +18,7 @@ class PythonSourceModule(object): kind = None - def __init__(self, name, path, logger, bytes_source = None): + def __init__(self, name, path, logger, diagnostics_writer, bytes_source = None): assert isinstance(path, str), path self.name = name # May be None self.path = path @@ -34,6 +34,7 @@ def __init__(self, name, path, logger, bytes_source = None): self._line_types = None self._comments = None self._tokens = None + self.diagnostics_writer = diagnostics_writer self.logger = logger with timers["decode"]: self.encoding, self.bytes_source = semmle.python.parser.tokenizer.encoding_from_source(bytes_source) @@ -113,6 +114,7 @@ def old_py_ast(self): self.logger.debug("Trying old parser on %s", self.path) self._py_ast = semmle.python.parser.parse(self.tokens, self.logger) self.logger.debug("Old parser successful on %s", self.path) + self.diagnostics_writer.record_old_parser() else: self.logger.debug("Found (during old_py_ast) parse tree for %s in cache", self.path) return self._py_ast @@ -147,6 +149,7 @@ def py_ast(self): self.logger.debug("Trying tsg-python on %s", self.path) self._py_ast = semmle.python.parser.tsg_parser.parse(self.path, self.logger) self.logger.debug("tsg-python successful on %s", self.path) + self.diagnostics_writer.record_tree_sitter_parser() else: self.logger.debug("Found (during py_ast) parse tree for %s in cache", self.path) return self._py_ast diff --git a/python/extractor/semmle/python/parser/dump_ast.py b/python/extractor/semmle/python/parser/dump_ast.py index 3a7db5ab0713..97abb502e59a 100644 --- a/python/extractor/semmle/python/parser/dump_ast.py +++ b/python/extractor/semmle/python/parser/dump_ast.py @@ -119,7 +119,8 @@ def reset_error_count(self): self.error_count = 0 def old_parser(inputfile, logger): - mod = PythonSourceModule(None, inputfile, logger) + from semmle.worker import DiagnosticsWriter + mod = PythonSourceModule(None, inputfile, logger, DiagnosticsWriter(0)) logger.close() return mod.old_py_ast diff --git a/python/extractor/semmle/python/passes/flow.py b/python/extractor/semmle/python/passes/flow.py index 6ea5405a8540..a100bbae7c9c 100755 --- a/python/extractor/semmle/python/passes/flow.py +++ b/python/extractor/semmle/python/passes/flow.py @@ -1916,7 +1916,8 @@ def write_ssa_phi(out, phi, arg): import semmle.python.parser.tsg_parser parsed_ast = semmle.python.parser.tsg_parser.parse(inputfile, FakeLogger()) else: - module = modules.PythonSourceModule("__main__", inputfile, FakeLogger()) + from semmle.worker import DiagnosticsWriter + module = modules.PythonSourceModule("__main__", inputfile, FakeLogger(), DiagnosticsWriter(0)) parsed_ast = module.ast FlowPass(options.split, options.prune, options.unroll).extract(parsed_ast, writer) writer.close() diff --git a/python/extractor/semmle/worker.py b/python/extractor/semmle/worker.py index 8d771828ada8..82997f186e4f 100644 --- a/python/extractor/semmle/worker.py +++ b/python/extractor/semmle/worker.py @@ -11,6 +11,7 @@ from semmle.profiling import get_profiler from semmle.path_rename import renamer_from_options_and_env from semmle.logging import WARN, recursion_error_message, internal_error_message, extractor_telemetry_message, Logger +from semmle.logging import parser_statistics_telemetry_message from semmle.util import FileExtractable, FolderExtractable class ExtractorFailure(Exception): @@ -245,9 +246,29 @@ def _write_extractor_telemetry(diagnostics_writer, logger: Logger): except OSError as ex: logger.warning("Failed to write extractor telemetry: %s", ex) +def _write_parser_statistics_telemetry(diagnostics_writer, logger: Logger): + counts = diagnostics_writer.parser_statistics() + if counts == (0, 0): + return + try: + diagnostics_writer.write(parser_statistics_telemetry_message(*counts)) + except OSError as ex: + logger.warning("Failed to write parser statistics telemetry: %s", ex) + class DiagnosticsWriter(object): def __init__(self, proc_id): self.proc_id = proc_id + self.old_parser_file_count = 0 + self.tree_sitter_parser_file_count = 0 + + def record_old_parser(self): + self.old_parser_file_count += 1 + + def record_tree_sitter_parser(self): + self.tree_sitter_parser_file_count += 1 + + def parser_statistics(self): + return self.old_parser_file_count, self.tree_sitter_parser_file_count def write(self, message): dir = os.environ.get("CODEQL_EXTRACTOR_PYTHON_DIAGNOSTIC_DIR") @@ -286,7 +307,7 @@ def _extract_loop(proc_id, queue, trap_dir, archive, options, reply_queue, logge _write_extractor_telemetry(diagnostics_writer, logger) try: if options.trace_only: - extractor = ModulePrinter(options, trap_dir, archive, renamer, logger) + extractor = ModulePrinter(options, trap_dir, archive, renamer, logger, diagnostics_writer) else: extractor = SuperExtractor(options, trap_dir, archive, renamer, logger, diagnostics_writer) profiler = get_profiler(options, id, logger) @@ -299,6 +320,7 @@ def _extract_loop(proc_id, queue, trap_dir, archive, options, reply_queue, logge if write_global_data: extractor.write_global_data() extractor.close() + _write_parser_statistics_telemetry(diagnostics_writer, logger) return try: start = time.time() @@ -352,4 +374,5 @@ def _extract_loop(proc_id, queue, trap_dir, archive, options, reply_queue, logge except _Empty: #Cleared queue enough to avoid deadlock. pass + _write_parser_statistics_telemetry(diagnostics_writer, logger) sys.exit(2) diff --git a/python/extractor/tests/test_diagnostics.py b/python/extractor/tests/test_diagnostics.py index a40f339e3e4b..7d0c2814ca68 100644 --- a/python/extractor/tests/test_diagnostics.py +++ b/python/extractor/tests/test_diagnostics.py @@ -3,6 +3,7 @@ from semmle import logging from semmle import util from semmle import worker +from semmle.python.modules import PythonSourceModule def test_extractor_telemetry_message(mocker): @@ -32,6 +33,32 @@ def test_extractor_telemetry_message(mocker): } +def test_parser_statistics_telemetry_message(): + message = logging.parser_statistics_telemetry_message( + old_parser_file_count=12, tree_sitter_parser_file_count=3 + ).to_dict() + message.pop("timestamp") + + assert message == { + "source": { + "id": "py/extractor/parser-statistics", + "name": "Python parser statistics", + "extractorName": "python", + }, + "severity": "note", + "markdownMessage": "Internal parser telemetry for the Python extractor.\n\nNo action needed.", + "visibility": { + "statusPage": False, + "cliSummaryTable": False, + "telemetry": True, + }, + "attributes": { + "old_parser_file_count": 12, + "tree_sitter_parser_file_count": 3, + }, + } + + def test_write_extractor_telemetry(mocker): diagnostics_writer = mocker.Mock() logger = mocker.Mock() @@ -39,6 +66,11 @@ def test_write_extractor_telemetry(mocker): worker._write_extractor_telemetry(diagnostics_writer, logger) diagnostics_writer.write.assert_called_once() + assert diagnostics_writer.write.call_args.args[0].to_dict()["attributes"] == { + "python_analysis_version": util.get_analysis_version(), + "python_runtime_version": platform.python_version(), + "extractor_version": util.VERSION, + } logger.warning.assert_not_called() @@ -52,3 +84,76 @@ def test_write_extractor_telemetry_handles_io_error(mocker): logger.warning.assert_called_once_with( "Failed to write extractor telemetry: %s", diagnostics_writer.write.side_effect ) + + +def test_write_parser_statistics_telemetry(mocker): + diagnostics_writer = mocker.Mock() + diagnostics_writer.parser_statistics.return_value = (1, 1) + logger = mocker.Mock() + + worker._write_parser_statistics_telemetry(diagnostics_writer, logger) + + diagnostics_writer.write.assert_called_once() + assert diagnostics_writer.write.call_args.args[0].to_dict()["attributes"] == { + "old_parser_file_count": 1, + "tree_sitter_parser_file_count": 1, + } + logger.warning.assert_not_called() + + +def test_does_not_write_empty_parser_statistics_telemetry(mocker): + diagnostics_writer = mocker.Mock() + diagnostics_writer.parser_statistics.return_value = (0, 0) + logger = mocker.Mock() + + worker._write_parser_statistics_telemetry(diagnostics_writer, logger) + + diagnostics_writer.write.assert_not_called() + logger.warning.assert_not_called() + + +def test_records_old_parser_usage_once(mocker, monkeypatch): + monkeypatch.delenv("CODEQL_PYTHON_DISABLE_OLD_PARSER", raising=False) + monkeypatch.delenv("CODEQL_PYTHON_DISABLE_TSG_PARSER", raising=False) + old_ast = object() + mocker.patch("semmle.python.parser.parse", return_value=old_ast) + diagnostics_writer = worker.DiagnosticsWriter(1) + module = PythonSourceModule( + None, + "test.py", + mocker.Mock(), + diagnostics_writer, + bytes_source=b"x = 1\n", + ) + + parsed_ast = module.py_ast + # Access the cached AST again to verify that it is not counted twice. + _ = module.py_ast + + assert parsed_ast is old_ast + assert diagnostics_writer.parser_statistics() == (1, 0) + + +def test_records_tree_sitter_parser_usage_once(mocker, monkeypatch): + monkeypatch.delenv("CODEQL_PYTHON_DISABLE_OLD_PARSER", raising=False) + monkeypatch.delenv("CODEQL_PYTHON_DISABLE_TSG_PARSER", raising=False) + tree_sitter_ast = object() + mocker.patch("semmle.python.parser.parse", side_effect=SyntaxError("old parser failed")) + mocker.patch( + "semmle.python.parser.tsg_parser.parse", return_value=tree_sitter_ast + ) + diagnostics_writer = worker.DiagnosticsWriter(1) + module = PythonSourceModule( + None, + "test.py", + mocker.Mock(), + diagnostics_writer, + bytes_source=b"x = 1\n", + ) + + parsed_ast = module.py_ast + # Access the cached AST again to verify that it is not counted twice. + _ = module.py_ast + + assert parsed_ast is tree_sitter_ast + assert diagnostics_writer.parser_statistics() == (0, 1) From e3dff5e42e94409e7dce7c16b6dddfed40b58b0c Mon Sep 17 00:00:00 2001 From: Taus Date: Thu, 3 Sep 2026 14:40:33 +0000 Subject: [PATCH 08/19] Python: Add extractor flag telemetry Records any non-default extractor flags (without their arguments) as a normalised string. This will enable us to determine which flags are actually used (and which ones we might therefore get rid of). When there are no flags other than the ones the autobuilder injects, we simply report the string `"default"`. That way, there's no need to remember exactly which flags are enabled by default during extraction. --- .../writing-diagnostics/diagnostics.expected | 1 + .../test_diagnostics_output.py | 14 ++++++++ python/extractor/semmle/cmdline.py | 17 ++++++++-- python/extractor/semmle/logging.py | 3 +- python/extractor/semmle/worker.py | 6 ++-- python/extractor/tests/test_cmdline.py | 34 +++++++++++++++++++ python/extractor/tests/test_diagnostics.py | 14 ++++++-- 7 files changed, 80 insertions(+), 9 deletions(-) create mode 100644 python/extractor/tests/test_cmdline.py diff --git a/python/extractor/cli-integration-test/writing-diagnostics/diagnostics.expected b/python/extractor/cli-integration-test/writing-diagnostics/diagnostics.expected index 715fb8dd12f7..4b6fceb48faf 100644 --- a/python/extractor/cli-integration-test/writing-diagnostics/diagnostics.expected +++ b/python/extractor/cli-integration-test/writing-diagnostics/diagnostics.expected @@ -163,6 +163,7 @@ } { "attributes": { + "extractor_flags": "default", "extractor_version": "7.1.10", "python_analysis_version": "3.12", "python_runtime_version": "3.12.3" diff --git a/python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py b/python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py index fae7bc93d88b..4108b9731943 100644 --- a/python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py +++ b/python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py @@ -1,9 +1,23 @@ import os import sys +import glob +import json sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..", "integration-tests")) import diagnostics_test_utils test_db = "db" +diagnostics = [] +diagnostic_dir = os.path.join(test_db, "diagnostic", "extractors", "python") +for path in glob.glob(os.path.join(diagnostic_dir, "*.jsonl")): + with open(path) as diagnostic_file: + diagnostics.extend(json.loads(line) for line in diagnostic_file) +summary = [ + diagnostic + for diagnostic in diagnostics + if diagnostic["source"]["id"] == "py/extractor/summary" +] +assert len(summary) == 1 +assert summary[0]["attributes"]["extractor_flags"] == "default" diagnostics_test_utils.check_diagnostics( ".", test_db, diff --git a/python/extractor/semmle/cmdline.py b/python/extractor/semmle/cmdline.py index 47007c065fdc..213a1f4affae 100644 --- a/python/extractor/semmle/cmdline.py +++ b/python/extractor/semmle/cmdline.py @@ -1,4 +1,4 @@ -from optparse import OptionParser, OptionGroup, HelpFormatter +from optparse import Option, OptionParser, OptionGroup, HelpFormatter import shlex import sys import os @@ -8,9 +8,21 @@ from semmle.util import VERSION +DEFAULT_AUTOBUILDER_FLAGS = {"R", "c", "v", "verbosity", "z"} + + +class RecordingOption(Option): + def process(self, opt, value, values, parser): + flag = (self._short_opts or self._long_opts)[0].lstrip("-") + if flag not in DEFAULT_AUTOBUILDER_FLAGS: + parser.extractor_flags.add(flag) + return Option.process(self, opt, value, values, parser) + + def make_parser(): '''Parse command_line, returning options, arguments''' - parser = OptionParser(add_help_option=False, version='%s' % VERSION) + parser = OptionParser(option_class=RecordingOption, add_help_option=False, version='%s' % VERSION) + parser.extractor_flags = set() import_options = OptionGroup(parser, "Import following options", description="Note that -a -n -g and -t are included for backwards compatibility. They are ignored") @@ -172,6 +184,7 @@ def parse(command_line): setattr(options, attr, dval) args.extend(extra_args) del options.file + options.extractor_flags = sorted(parser.extractor_flags) if options.help: if options.verbose: for opt in parser._get_all_options(): diff --git a/python/extractor/semmle/logging.py b/python/extractor/semmle/logging.py index 6f5255a7a9d5..652ab810d0fb 100644 --- a/python/extractor/semmle/logging.py +++ b/python/extractor/semmle/logging.py @@ -358,12 +358,13 @@ def with_timestamp(self, timestamp): self.timestamp = timestamp return self -def extractor_telemetry_message(): +def extractor_telemetry_message(extractor_flags): return (DiagnosticMessage(Source("py/extractor/summary", "Python extractor telemetry"), Severity.NOTE) .markdown("Internal telemetry for the Python extractor.\n\nNo action needed.") .attribute("python_analysis_version", get_analysis_version()) .attribute("python_runtime_version", platform.python_version()) .attribute("extractor_version", VERSION) + .attribute("extractor_flags", " ".join(extractor_flags) or "default") .telemetry() ) diff --git a/python/extractor/semmle/worker.py b/python/extractor/semmle/worker.py index 82997f186e4f..15b7b556bf88 100644 --- a/python/extractor/semmle/worker.py +++ b/python/extractor/semmle/worker.py @@ -240,9 +240,9 @@ def _drain_queue(queue): #Emptied queue as best we can. pass -def _write_extractor_telemetry(diagnostics_writer, logger: Logger): +def _write_extractor_telemetry(diagnostics_writer, logger: Logger, extractor_flags): try: - diagnostics_writer.write(extractor_telemetry_message()) + diagnostics_writer.write(extractor_telemetry_message(extractor_flags)) except OSError as ex: logger.warning("Failed to write extractor telemetry: %s", ex) @@ -304,7 +304,7 @@ def _extract_loop(proc_id, queue, trap_dir, archive, options, reply_queue, logge sys.exit(2) logger.set_process_id(proc_id) if write_global_data: - _write_extractor_telemetry(diagnostics_writer, logger) + _write_extractor_telemetry(diagnostics_writer, logger, options.extractor_flags) try: if options.trace_only: extractor = ModulePrinter(options, trap_dir, archive, renamer, logger, diagnostics_writer) diff --git a/python/extractor/tests/test_cmdline.py b/python/extractor/tests/test_cmdline.py new file mode 100644 index 000000000000..76f2fe951b57 --- /dev/null +++ b/python/extractor/tests/test_cmdline.py @@ -0,0 +1,34 @@ +from semmle import cmdline + + +def test_records_flags_without_values(): + options, args = cmdline.parse( + [ + "--verbosity=3", + "-zall", + "-R", + "/src", + "-vv", + "--path", + "/lib", + "-p", + "/other-lib", + "module", + ] + ) + + assert options.extractor_flags == ["p"] + assert args == ["module"] + + +def test_records_flags_from_option_file(tmp_path): + options_file = tmp_path / "extractor-options" + options_file.write_text("--colorize --max-import-depth 2") + + options, _ = cmdline.parse(["-f", str(options_file)]) + + assert options.extractor_flags == [ + "colorize", + "f", + "max-import-depth", + ] diff --git a/python/extractor/tests/test_diagnostics.py b/python/extractor/tests/test_diagnostics.py index 7d0c2814ca68..ddfd22436de9 100644 --- a/python/extractor/tests/test_diagnostics.py +++ b/python/extractor/tests/test_diagnostics.py @@ -9,7 +9,7 @@ def test_extractor_telemetry_message(mocker): mocker.patch("semmle.logging.get_analysis_version", return_value="3.13") - message = logging.extractor_telemetry_message().to_dict() + message = logging.extractor_telemetry_message(["colorize", "p"]).to_dict() message.pop("timestamp") assert message == { @@ -29,6 +29,7 @@ def test_extractor_telemetry_message(mocker): "python_analysis_version": "3.13", "python_runtime_version": platform.python_version(), "extractor_version": util.VERSION, + "extractor_flags": "colorize p", }, } @@ -59,17 +60,24 @@ def test_parser_statistics_telemetry_message(): } +def test_extractor_telemetry_message_includes_empty_flags(): + message = logging.extractor_telemetry_message([]).to_dict() + + assert message["attributes"]["extractor_flags"] == "default" + + def test_write_extractor_telemetry(mocker): diagnostics_writer = mocker.Mock() logger = mocker.Mock() - worker._write_extractor_telemetry(diagnostics_writer, logger) + worker._write_extractor_telemetry(diagnostics_writer, logger, ["quiet"]) diagnostics_writer.write.assert_called_once() assert diagnostics_writer.write.call_args.args[0].to_dict()["attributes"] == { "python_analysis_version": util.get_analysis_version(), "python_runtime_version": platform.python_version(), "extractor_version": util.VERSION, + "extractor_flags": "quiet", } logger.warning.assert_not_called() @@ -79,7 +87,7 @@ def test_write_extractor_telemetry_handles_io_error(mocker): diagnostics_writer.write.side_effect = OSError("write failed") logger = mocker.Mock() - worker._write_extractor_telemetry(diagnostics_writer, logger) + worker._write_extractor_telemetry(diagnostics_writer, logger, []) logger.warning.assert_called_once_with( "Failed to write extractor telemetry: %s", diagnostics_writer.write.side_effect From cb6264182606cd012e0c353a30e0745b10fe58fe Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 3 Sep 2026 22:41:48 +0000 Subject: [PATCH 09/19] Post-release preparation for codeql-cli-2.27.0 --- actions/ql/lib/qlpack.yml | 2 +- actions/ql/src/qlpack.yml | 2 +- cpp/ql/lib/qlpack.yml | 2 +- cpp/ql/src/qlpack.yml | 2 +- csharp/ql/campaigns/Solorigate/lib/qlpack.yml | 2 +- csharp/ql/campaigns/Solorigate/src/qlpack.yml | 2 +- csharp/ql/lib/qlpack.yml | 2 +- csharp/ql/src/qlpack.yml | 2 +- go/ql/consistency-queries/qlpack.yml | 2 +- go/ql/lib/qlpack.yml | 2 +- go/ql/src/qlpack.yml | 2 +- java/ql/lib/qlpack.yml | 2 +- java/ql/src/qlpack.yml | 2 +- javascript/ql/lib/qlpack.yml | 2 +- javascript/ql/src/qlpack.yml | 2 +- misc/suite-helpers/qlpack.yml | 2 +- python/ql/lib/qlpack.yml | 2 +- python/ql/src/qlpack.yml | 2 +- ruby/ql/lib/qlpack.yml | 2 +- ruby/ql/src/qlpack.yml | 2 +- rust/ql/lib/qlpack.yml | 2 +- rust/ql/src/qlpack.yml | 2 +- shared/concepts/qlpack.yml | 2 +- shared/controlflow/qlpack.yml | 2 +- shared/dataflow/qlpack.yml | 2 +- shared/mad/qlpack.yml | 2 +- shared/namebinding/qlpack.yml | 2 +- shared/quantum/qlpack.yml | 2 +- shared/rangeanalysis/qlpack.yml | 2 +- shared/regex/qlpack.yml | 2 +- shared/ssa/qlpack.yml | 2 +- shared/threat-models/qlpack.yml | 2 +- shared/tutorial/qlpack.yml | 2 +- shared/typeflow/qlpack.yml | 2 +- shared/typeinference/qlpack.yml | 2 +- shared/typetracking/qlpack.yml | 2 +- shared/typos/qlpack.yml | 2 +- shared/util/qlpack.yml | 2 +- shared/xml/qlpack.yml | 2 +- shared/yaml/qlpack.yml | 2 +- swift/ql/lib/qlpack.yml | 2 +- swift/ql/src/qlpack.yml | 2 +- 42 files changed, 42 insertions(+), 42 deletions(-) diff --git a/actions/ql/lib/qlpack.yml b/actions/ql/lib/qlpack.yml index 612e631c6f8b..268018a37415 100644 --- a/actions/ql/lib/qlpack.yml +++ b/actions/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/actions-all -version: 0.6.1 +version: 0.6.2-dev library: true warnOnImplicitThis: true dependencies: diff --git a/actions/ql/src/qlpack.yml b/actions/ql/src/qlpack.yml index 5e4877698e63..5feedb52c8ce 100644 --- a/actions/ql/src/qlpack.yml +++ b/actions/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/actions-queries -version: 0.6.35 +version: 0.6.36-dev library: false warnOnImplicitThis: true groups: [actions, queries] diff --git a/cpp/ql/lib/qlpack.yml b/cpp/ql/lib/qlpack.yml index c3e40cb63948..5114bf861a36 100644 --- a/cpp/ql/lib/qlpack.yml +++ b/cpp/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/cpp-all -version: 12.1.0 +version: 12.1.1-dev groups: cpp dbscheme: semmlecode.cpp.dbscheme extractor: cpp diff --git a/cpp/ql/src/qlpack.yml b/cpp/ql/src/qlpack.yml index ca971b04aaad..034523449c24 100644 --- a/cpp/ql/src/qlpack.yml +++ b/cpp/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/cpp-queries -version: 1.8.3 +version: 1.8.4-dev groups: - cpp - queries diff --git a/csharp/ql/campaigns/Solorigate/lib/qlpack.yml b/csharp/ql/campaigns/Solorigate/lib/qlpack.yml index 990c6ad4dbc8..07c306a65a9d 100644 --- a/csharp/ql/campaigns/Solorigate/lib/qlpack.yml +++ b/csharp/ql/campaigns/Solorigate/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-solorigate-all -version: 1.7.74 +version: 1.7.75-dev groups: - csharp - solorigate diff --git a/csharp/ql/campaigns/Solorigate/src/qlpack.yml b/csharp/ql/campaigns/Solorigate/src/qlpack.yml index 199504aa7dd3..94bf9c0f7db3 100644 --- a/csharp/ql/campaigns/Solorigate/src/qlpack.yml +++ b/csharp/ql/campaigns/Solorigate/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-solorigate-queries -version: 1.7.74 +version: 1.7.75-dev groups: - csharp - solorigate diff --git a/csharp/ql/lib/qlpack.yml b/csharp/ql/lib/qlpack.yml index 15fadfad8a0f..18ee2c149098 100644 --- a/csharp/ql/lib/qlpack.yml +++ b/csharp/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-all -version: 7.3.0 +version: 7.3.1-dev groups: csharp dbscheme: semmlecode.csharp.dbscheme extractor: csharp diff --git a/csharp/ql/src/qlpack.yml b/csharp/ql/src/qlpack.yml index a2f6f0aac243..c90064a3b913 100644 --- a/csharp/ql/src/qlpack.yml +++ b/csharp/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/csharp-queries -version: 1.9.3 +version: 1.9.4-dev groups: - csharp - queries diff --git a/go/ql/consistency-queries/qlpack.yml b/go/ql/consistency-queries/qlpack.yml index 353e4dd1cdcd..c9c318de2fa2 100644 --- a/go/ql/consistency-queries/qlpack.yml +++ b/go/ql/consistency-queries/qlpack.yml @@ -1,5 +1,5 @@ name: codeql-go-consistency-queries -version: 1.0.57 +version: 1.0.58-dev groups: - go - queries diff --git a/go/ql/lib/qlpack.yml b/go/ql/lib/qlpack.yml index fb42dcb703ae..4dc828abeffe 100644 --- a/go/ql/lib/qlpack.yml +++ b/go/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/go-all -version: 7.3.1 +version: 7.3.2-dev groups: go dbscheme: go.dbscheme extractor: go diff --git a/go/ql/src/qlpack.yml b/go/ql/src/qlpack.yml index bc9f243309af..fdfdd1f67432 100644 --- a/go/ql/src/qlpack.yml +++ b/go/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/go-queries -version: 1.6.10 +version: 1.6.11-dev groups: - go - queries diff --git a/java/ql/lib/qlpack.yml b/java/ql/lib/qlpack.yml index d97a58c5c809..d6aefad45233 100644 --- a/java/ql/lib/qlpack.yml +++ b/java/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/java-all -version: 9.3.0 +version: 9.3.1-dev groups: java dbscheme: config/semmlecode.dbscheme extractor: java diff --git a/java/ql/src/qlpack.yml b/java/ql/src/qlpack.yml index 9c3cffa047ca..1bb0cdc16c09 100644 --- a/java/ql/src/qlpack.yml +++ b/java/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/java-queries -version: 1.11.10 +version: 1.11.11-dev groups: - java - queries diff --git a/javascript/ql/lib/qlpack.yml b/javascript/ql/lib/qlpack.yml index 97f274ff58ae..93c4370aab6d 100644 --- a/javascript/ql/lib/qlpack.yml +++ b/javascript/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/javascript-all -version: 2.10.1 +version: 2.10.2-dev groups: javascript dbscheme: semmlecode.javascript.dbscheme extractor: javascript diff --git a/javascript/ql/src/qlpack.yml b/javascript/ql/src/qlpack.yml index 83e2caff741c..456c7f4ca7b5 100644 --- a/javascript/ql/src/qlpack.yml +++ b/javascript/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/javascript-queries -version: 2.4.5 +version: 2.4.6-dev groups: - javascript - queries diff --git a/misc/suite-helpers/qlpack.yml b/misc/suite-helpers/qlpack.yml index ba51f45be36f..1c27d6177555 100644 --- a/misc/suite-helpers/qlpack.yml +++ b/misc/suite-helpers/qlpack.yml @@ -1,4 +1,4 @@ name: codeql/suite-helpers -version: 1.0.57 +version: 1.0.58-dev groups: shared warnOnImplicitThis: true diff --git a/python/ql/lib/qlpack.yml b/python/ql/lib/qlpack.yml index 715a4f61bfc8..ad9d7605202a 100644 --- a/python/ql/lib/qlpack.yml +++ b/python/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/python-all -version: 7.2.5 +version: 7.2.6-dev groups: python dbscheme: semmlecode.python.dbscheme extractor: python diff --git a/python/ql/src/qlpack.yml b/python/ql/src/qlpack.yml index d48bc1e8e7cc..70ce9eeef942 100644 --- a/python/ql/src/qlpack.yml +++ b/python/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/python-queries -version: 1.8.10 +version: 1.8.11-dev groups: - python - queries diff --git a/ruby/ql/lib/qlpack.yml b/ruby/ql/lib/qlpack.yml index 2c52daba2b15..f38a7120a31f 100644 --- a/ruby/ql/lib/qlpack.yml +++ b/ruby/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/ruby-all -version: 7.0.0 +version: 7.0.1-dev groups: ruby extractor: ruby dbscheme: ruby.dbscheme diff --git a/ruby/ql/src/qlpack.yml b/ruby/ql/src/qlpack.yml index a215aec72b83..d532c03248a0 100644 --- a/ruby/ql/src/qlpack.yml +++ b/ruby/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/ruby-queries -version: 1.6.10 +version: 1.6.11-dev groups: - ruby - queries diff --git a/rust/ql/lib/qlpack.yml b/rust/ql/lib/qlpack.yml index dc1059ff64c5..bb31a181ec16 100644 --- a/rust/ql/lib/qlpack.yml +++ b/rust/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/rust-all -version: 0.2.21 +version: 0.2.22-dev groups: rust extractor: rust dbscheme: rust.dbscheme diff --git a/rust/ql/src/qlpack.yml b/rust/ql/src/qlpack.yml index 2e13de282d58..dc4e53aeaf81 100644 --- a/rust/ql/src/qlpack.yml +++ b/rust/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/rust-queries -version: 0.1.42 +version: 0.1.43-dev groups: - rust - queries diff --git a/shared/concepts/qlpack.yml b/shared/concepts/qlpack.yml index f5005081b57a..677235474b91 100644 --- a/shared/concepts/qlpack.yml +++ b/shared/concepts/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/concepts -version: 0.0.31 +version: 0.0.32-dev groups: shared library: true dependencies: diff --git a/shared/controlflow/qlpack.yml b/shared/controlflow/qlpack.yml index fd5bfcb78586..66ae31f61364 100644 --- a/shared/controlflow/qlpack.yml +++ b/shared/controlflow/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/controlflow -version: 2.0.41 +version: 2.0.42-dev groups: shared library: true dependencies: diff --git a/shared/dataflow/qlpack.yml b/shared/dataflow/qlpack.yml index 09e851c73c93..134205735fe7 100644 --- a/shared/dataflow/qlpack.yml +++ b/shared/dataflow/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/dataflow -version: 2.1.13 +version: 2.1.14-dev groups: shared library: true dependencies: diff --git a/shared/mad/qlpack.yml b/shared/mad/qlpack.yml index e350a3a2ddbd..783f372bc478 100644 --- a/shared/mad/qlpack.yml +++ b/shared/mad/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/mad -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true dependencies: diff --git a/shared/namebinding/qlpack.yml b/shared/namebinding/qlpack.yml index f18af62921c4..af4358254944 100644 --- a/shared/namebinding/qlpack.yml +++ b/shared/namebinding/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/namebinding -version: 0.0.6 +version: 0.0.7-dev groups: shared library: true dependencies: diff --git a/shared/quantum/qlpack.yml b/shared/quantum/qlpack.yml index 83384e926bcb..044e243531fd 100644 --- a/shared/quantum/qlpack.yml +++ b/shared/quantum/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/quantum -version: 0.0.35 +version: 0.0.36-dev groups: shared library: true dependencies: diff --git a/shared/rangeanalysis/qlpack.yml b/shared/rangeanalysis/qlpack.yml index 2b0eb43866da..6b4e8a5adede 100644 --- a/shared/rangeanalysis/qlpack.yml +++ b/shared/rangeanalysis/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/rangeanalysis -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true dependencies: diff --git a/shared/regex/qlpack.yml b/shared/regex/qlpack.yml index d4ad24634326..ad414ca86d3d 100644 --- a/shared/regex/qlpack.yml +++ b/shared/regex/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/regex -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true dependencies: diff --git a/shared/ssa/qlpack.yml b/shared/ssa/qlpack.yml index 92b8603a9d6e..07e6041a5142 100644 --- a/shared/ssa/qlpack.yml +++ b/shared/ssa/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/ssa -version: 2.0.33 +version: 2.0.34-dev groups: shared library: true dependencies: diff --git a/shared/threat-models/qlpack.yml b/shared/threat-models/qlpack.yml index a51fdda87b7f..84c172ce3f25 100644 --- a/shared/threat-models/qlpack.yml +++ b/shared/threat-models/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/threat-models -version: 1.0.57 +version: 1.0.58-dev library: true groups: shared dataExtensions: diff --git a/shared/tutorial/qlpack.yml b/shared/tutorial/qlpack.yml index 2c2dde8c1793..fa374cda8f72 100644 --- a/shared/tutorial/qlpack.yml +++ b/shared/tutorial/qlpack.yml @@ -1,7 +1,7 @@ name: codeql/tutorial description: Library for the CodeQL detective tutorials, helping new users learn to write CodeQL queries. -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true warnOnImplicitThis: true diff --git a/shared/typeflow/qlpack.yml b/shared/typeflow/qlpack.yml index 6ea63f761cdd..4dd13d3aad94 100644 --- a/shared/typeflow/qlpack.yml +++ b/shared/typeflow/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typeflow -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true dependencies: diff --git a/shared/typeinference/qlpack.yml b/shared/typeinference/qlpack.yml index 5cabb023ce1e..68db81a4851b 100644 --- a/shared/typeinference/qlpack.yml +++ b/shared/typeinference/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typeinference -version: 0.0.38 +version: 0.0.39-dev groups: shared library: true dependencies: diff --git a/shared/typetracking/qlpack.yml b/shared/typetracking/qlpack.yml index 854d8bae6da0..9b52023e6343 100644 --- a/shared/typetracking/qlpack.yml +++ b/shared/typetracking/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typetracking -version: 2.0.41 +version: 2.0.42-dev groups: shared library: true dependencies: diff --git a/shared/typos/qlpack.yml b/shared/typos/qlpack.yml index 712073d7668a..ade80e7678f0 100644 --- a/shared/typos/qlpack.yml +++ b/shared/typos/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/typos -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true warnOnImplicitThis: true diff --git a/shared/util/qlpack.yml b/shared/util/qlpack.yml index 76bb1b6957a2..0a61f3903159 100644 --- a/shared/util/qlpack.yml +++ b/shared/util/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/util -version: 2.0.44 +version: 2.0.45-dev groups: shared library: true dependencies: null diff --git a/shared/xml/qlpack.yml b/shared/xml/qlpack.yml index c9251eb88c4d..133a3b9dac86 100644 --- a/shared/xml/qlpack.yml +++ b/shared/xml/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/xml -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true dependencies: diff --git a/shared/yaml/qlpack.yml b/shared/yaml/qlpack.yml index ccc990b27d02..14b67df2f8d1 100644 --- a/shared/yaml/qlpack.yml +++ b/shared/yaml/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/yaml -version: 1.0.57 +version: 1.0.58-dev groups: shared library: true warnOnImplicitThis: true diff --git a/swift/ql/lib/qlpack.yml b/swift/ql/lib/qlpack.yml index dd31b62e1481..9806fded2032 100644 --- a/swift/ql/lib/qlpack.yml +++ b/swift/ql/lib/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/swift-all -version: 6.8.3 +version: 6.8.4-dev groups: swift extractor: swift dbscheme: swift.dbscheme diff --git a/swift/ql/src/qlpack.yml b/swift/ql/src/qlpack.yml index fd6ec0e549bc..aa419cb33887 100644 --- a/swift/ql/src/qlpack.yml +++ b/swift/ql/src/qlpack.yml @@ -1,5 +1,5 @@ name: codeql/swift-queries -version: 1.3.10 +version: 1.3.11-dev groups: - swift - queries From da77b35c728ee767655cbdbb5e3aa42d1261dc4d Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Thu, 27 Aug 2026 12:51:35 +0000 Subject: [PATCH 10/19] Hoist identical fields from union members to abstract predicates Co-authored-by: aschackmull <28296824+aschackmull@users.noreply.github.com> --- .../tree-sitter-extractor/src/generator/ql.rs | 14 +- .../src/generator/ql_gen.rs | 315 +++++++++++++----- 2 files changed, 249 insertions(+), 80 deletions(-) diff --git a/shared/tree-sitter-extractor/src/generator/ql.rs b/shared/tree-sitter-extractor/src/generator/ql.rs index f114e251af21..2d91d6c66f67 100644 --- a/shared/tree-sitter-extractor/src/generator/ql.rs +++ b/shared/tree-sitter-extractor/src/generator/ql.rs @@ -109,7 +109,7 @@ impl fmt::Display for Class<'_> { is_final: false, return_type: None, formal_parameters: vec![], - body: charpred.clone(), + body: Some(charpred.clone()), overlay: None, } )?; @@ -307,7 +307,9 @@ pub struct Predicate<'a> { pub is_final: bool, pub return_type: Option>, pub formal_parameters: Vec>, - pub body: Expression<'a>, + /// The body of the predicate, or `None` if this is an `abstract` + /// predicate declaration with no body. + pub body: Option>, pub overlay: Option, } @@ -330,6 +332,9 @@ impl fmt::Display for Predicate<'_> { if self.is_final { write!(f, "final ")?; } + if self.body.is_none() { + write!(f, "abstract ")?; + } if self.overridden { write!(f, "override ")?; } @@ -344,7 +349,10 @@ impl fmt::Display for Predicate<'_> { } write!(f, "{param}")?; } - write!(f, ") {{ {} }}", self.body)?; + match &self.body { + Some(body) => write!(f, ") {{ {body} }}")?, + None => write!(f, ");")?, + } Ok(()) } diff --git a/shared/tree-sitter-extractor/src/generator/ql_gen.rs b/shared/tree-sitter-extractor/src/generator/ql_gen.rs index 237ed9ddb968..04a9e73aee1b 100644 --- a/shared/tree-sitter-extractor/src/generator/ql_gen.rs +++ b/shared/tree-sitter-extractor/src/generator/ql_gen.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeMap; use std::collections::BTreeSet; use crate::{generator::ql, node_types}; @@ -20,14 +21,14 @@ pub fn create_ast_node_class<'a>( is_final: false, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: ql::Expression::Equals( + body: Some(ql::Expression::Equals( Box::new(ql::Expression::Var("result")), Box::new(ql::Expression::Dot( Box::new(ql::Expression::Var("this")), "getAPrimaryQlClass", vec![], )), - ), + )), overlay: None, }; let get_location = ql::Predicate { @@ -38,10 +39,10 @@ pub fn create_ast_node_class<'a>( is_final: true, return_type: Some(ql::Type::Normal("L::Location")), formal_parameters: vec![], - body: ql::Expression::Pred( + body: Some(ql::Expression::Pred( node_location_table, vec![ql::Expression::Var("this"), ql::Expression::Var("result")], - ), + )), overlay: None, }; let get_a_field_or_child = create_none_predicate( @@ -58,14 +59,14 @@ pub fn create_ast_node_class<'a>( is_final: true, return_type: Some(ql::Type::Facade("AstNode")), formal_parameters: vec![], - body: ql::Expression::Pred( + body: Some(ql::Expression::Pred( node_parent_table, vec![ ql::Expression::Var("this"), ql::Expression::Var("result"), ql::Expression::Var("_"), ], - ), + )), overlay: None, }; let get_parent_index = ql::Predicate { @@ -78,14 +79,14 @@ pub fn create_ast_node_class<'a>( is_final: true, return_type: Some(ql::Type::Int), formal_parameters: vec![], - body: ql::Expression::Pred( + body: Some(ql::Expression::Pred( node_parent_table, vec![ ql::Expression::Var("this"), ql::Expression::Var("_"), ql::Expression::Var("result"), ], - ), + )), overlay: None, }; let get_a_primary_ql_class = ql::Predicate { @@ -98,10 +99,10 @@ pub fn create_ast_node_class<'a>( is_final: false, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: ql::Expression::Equals( + body: Some(ql::Expression::Equals( Box::new(ql::Expression::Var("result")), Box::new(ql::Expression::String("???")), - ), + )), overlay: None, }; let get_primary_ql_classes = ql::Predicate { @@ -116,7 +117,7 @@ pub fn create_ast_node_class<'a>( is_final: false, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: ql::Expression::Equals( + body: Some(ql::Expression::Equals( Box::new(ql::Expression::Var("result")), Box::new(ql::Expression::Aggregate { name: "concat", @@ -129,7 +130,7 @@ pub fn create_ast_node_class<'a>( )), second_expr: Some(Box::new(ql::Expression::String(","))), }), - ), + )), overlay: None, }; ql::Class { @@ -163,7 +164,12 @@ pub fn create_token_class<'a>(token_type: &'a str, tokeninfo: &'a str) -> ql::Cl is_final: true, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: create_get_field_expr_for_column_storage("result", tokeninfo, 1, tokeninfo_arity), + body: Some(create_get_field_expr_for_column_storage( + "result", + tokeninfo, + 1, + tokeninfo_arity, + )), overlay: None, }; let to_string = ql::Predicate { @@ -176,14 +182,14 @@ pub fn create_token_class<'a>(token_type: &'a str, tokeninfo: &'a str) -> ql::Cl is_final: true, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: ql::Expression::Equals( + body: Some(ql::Expression::Equals( Box::new(ql::Expression::Var("result")), Box::new(ql::Expression::Dot( Box::new(ql::Expression::Var("this")), "getValue", vec![], )), - ), + )), overlay: None, }; ql::Class { @@ -223,12 +229,12 @@ pub fn create_trivia_token_class<'a>( is_final: true, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: create_get_field_expr_for_column_storage( + body: Some(create_get_field_expr_for_column_storage( "result", trivia_tokeninfo, 1, trivia_tokeninfo_arity, - ), + )), overlay: None, }; let to_string = ql::Predicate { @@ -241,14 +247,14 @@ pub fn create_trivia_token_class<'a>( is_final: true, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: ql::Expression::Equals( + body: Some(ql::Expression::Equals( Box::new(ql::Expression::Var("result")), Box::new(ql::Expression::Dot( Box::new(ql::Expression::Var("this")), "getValue", vec![], )), - ), + )), overlay: None, }; ql::Class { @@ -306,7 +312,7 @@ fn create_none_predicate<'a>( is_final: false, return_type, formal_parameters: Vec::new(), - body: ql::Expression::Pred("none", vec![]), + body: Some(ql::Expression::Pred("none", vec![])), overlay: None, } } @@ -324,10 +330,10 @@ fn create_get_a_primary_ql_class(class_name: &str, is_final: bool) -> ql::Predic is_final, return_type: Some(ql::Type::String), formal_parameters: vec![], - body: ql::Expression::Equals( + body: Some(ql::Expression::Equals( Box::new(ql::Expression::Var("result")), Box::new(ql::Expression::String(class_name)), - ), + )), overlay: None, } } @@ -342,13 +348,13 @@ pub fn create_is_overlay_predicate() -> ql::Predicate<'static> { return_type: None, overlay: Some(ql::OverlayAnnotation::Local), formal_parameters: vec![], - body: ql::Expression::Pred( + body: Some(ql::Expression::Pred( "databaseMetadata", vec![ ql::Expression::String("isOverlay"), ql::Expression::String("true"), ], - ), + )), } } @@ -368,7 +374,7 @@ pub fn create_get_node_file_predicate<'a>( name: "node", param_type: ql::Type::At(ast_node_name), }], - body: ql::Expression::Aggregate { + body: Some(ql::Expression::Aggregate { name: "exists", vars: vec![ql::FormalParameter { name: "loc", @@ -390,7 +396,7 @@ pub fn create_get_node_file_predicate<'a>( ], )), second_expr: None, - }, + }), } } @@ -415,7 +421,7 @@ pub fn create_discardable_ast_node_predicate(ast_node_name: &str) -> ql::Predica param_type: ql::Type::At(ast_node_name), }, ], - body: ql::Expression::And(vec![ + body: Some(ql::Expression::And(vec![ ql::Expression::Negation(Box::new(ql::Expression::Pred("isOverlay", vec![]))), ql::Expression::Equals( Box::new(ql::Expression::Var("file")), @@ -424,7 +430,7 @@ pub fn create_discardable_ast_node_predicate(ast_node_name: &str) -> ql::Predica vec![ql::Expression::Var("node")], )), ), - ]), + ])), } } @@ -444,7 +450,7 @@ pub fn create_discard_ast_node_predicate(ast_node_name: &str) -> ql::Predicate<' name: "node", param_type: ql::Type::At(ast_node_name), }], - body: ql::Expression::Aggregate { + body: Some(ql::Expression::Aggregate { name: "exists", vars: vec![ ql::FormalParameter { @@ -468,7 +474,7 @@ pub fn create_discard_ast_node_predicate(ast_node_name: &str) -> ql::Predicate<' ql::Expression::Pred("overlayChangedFiles", vec![ql::Expression::Var("path")]), ])), second_expr: None, - }, + }), } } @@ -493,7 +499,7 @@ pub fn create_discardable_location_predicate() -> ql::Predicate<'static> { param_type: ql::Type::At("location_default"), }, ], - body: ql::Expression::And(vec![ + body: Some(ql::Expression::And(vec![ ql::Expression::Negation(Box::new(ql::Expression::Pred("isOverlay", vec![]))), ql::Expression::Pred( "locations_default", @@ -506,7 +512,7 @@ pub fn create_discardable_location_predicate() -> ql::Predicate<'static> { ql::Expression::Var("_"), ], ), - ]), + ])), } } @@ -529,7 +535,7 @@ pub fn create_discard_location_predicate() -> ql::Predicate<'static> { name: "loc", param_type: ql::Type::At("location_default"), }], - body: ql::Expression::Aggregate { + body: Some(ql::Expression::Aggregate { name: "exists", vars: vec![ ql::FormalParameter { @@ -553,7 +559,7 @@ pub fn create_discard_location_predicate() -> ql::Predicate<'static> { ql::Expression::Pred("overlayChangedFiles", vec![ql::Expression::Var("path")]), ])), second_expr: None, - }, + }), } } @@ -760,7 +766,7 @@ fn create_field_getters<'a>( is_final: true, return_type: return_type.clone(), formal_parameters, - body, + body: Some(body), overlay: None, }]; @@ -773,14 +779,14 @@ fn create_field_getters<'a>( is_final: true, return_type, formal_parameters: vec![], - body: ql::Expression::Equals( + body: Some(ql::Expression::Equals( Box::new(ql::Expression::Var("result")), Box::new(ql::Expression::Dot( Box::new(ql::Expression::Var("this")), &field.getter_name, vec![ql::Expression::Var("_")], )), - ), + )), overlay: None, }); } @@ -828,6 +834,86 @@ fn class_supertypes<'a>( supertypes } +/// Returns whether `a` and `b` have the same signature, i.e. the same name, +/// return type, and formal parameters. Predicates with the same signature can +/// override one another. +fn same_predicate_signature(a: &ql::Predicate, b: &ql::Predicate) -> bool { + a.name == b.name && a.return_type == b.return_type && a.formal_parameters == b.formal_parameters +} + +/// Computes, for each tree-sitter supertype (union) node, the list of +/// predicates that are guaranteed to be defined identically (in terms of +/// name, return type, and formal parameters, though not necessarily body) by +/// every one of its members. These are the predicates that can be hoisted to +/// an `abstract` predicate on the union's class, with the corresponding +/// predicates on its members becoming `override`s. +/// +/// The result for a given node is memoized in `cache` (keyed by its QL class +/// name), and also used to answer the query for any other node that +/// (directly, or transitively through further supertypes) has that node as a +/// member. The same cache also serves as the answer to "what does the class +/// named X expose?", used by `is_predicate_inherited`. +fn compute_exposed_predicates<'a, 'b>( + type_name: &'a node_types::TypeName, + nodes: &'a node_types::NodeTypeMap, + field_predicates: &BTreeMap<&node_types::TypeName, Vec>>, + cache: &'b mut BTreeMap<&'a str, Vec>>, +) -> &'b Vec> { + let node = nodes.get(type_name); + let class_name = node.map_or(type_name.kind.as_str(), |node| node.ql_class_name.as_str()); + if !cache.contains_key(class_name) { + let exposed = match node.map(|node| &node.kind) { + Some(node_types::EntryKind::Table { .. }) => { + field_predicates.get(type_name).cloned().unwrap_or_default() + } + Some(node_types::EntryKind::Union { members }) => { + let mut members = members.iter(); + let mut common = match members.next() { + Some(first) => { + compute_exposed_predicates(first, nodes, field_predicates, cache).clone() + } + None => Vec::new(), + }; + for member in members { + let member_predicates = + compute_exposed_predicates(member, nodes, field_predicates, cache); + common.retain(|predicate| { + member_predicates + .iter() + .any(|other| same_predicate_signature(predicate, other)) + }); + } + common + } + Some(node_types::EntryKind::Token { .. }) | None => Vec::new(), + }; + cache.insert(class_name, exposed); + } + cache.get(class_name).unwrap() +} + +/// Returns whether `predicate` (declared, or about to be declared, on the +/// class for `type_name`) is already exposed by one of `type_name`'s direct +/// supertypes, and therefore must be marked as an `override` (for a concrete +/// predicate) or can be omitted entirely (for an `abstract` one, since it's +/// already inherited). +fn is_predicate_inherited( + predicate: &ql::Predicate, + type_name: &node_types::TypeName, + direct_supertypes: &BTreeMap>, + exposed_predicates: &BTreeMap<&str, Vec>, +) -> bool { + direct_supertypes.get(type_name).is_some_and(|supertypes| { + supertypes.iter().any(|supertype| { + exposed_predicates.get(supertype).is_some_and(|predicates| { + predicates + .iter() + .any(|other| same_predicate_signature(predicate, other)) + }) + }) + }) +} + /// Converts the given node types into CodeQL classes wrapping the dbscheme. pub fn convert_nodes(nodes: &node_types::NodeTypeMap) -> Vec> { let mut classes = Vec::new(); @@ -841,6 +927,71 @@ pub fn convert_nodes(nodes: &node_types::NodeTypeMap) -> Vec> { } } + // First, compute the field-getter predicates (and the expressions used by + // `getAFieldOrChild`) for every table node, without yet knowing whether + // any of them will need to be marked `override`. These are needed both + // to build the final classes below, and to figure out which fields are + // shared identically by all the members of a supertype. + let mut field_predicates: BTreeMap<&node_types::TypeName, Vec>> = + BTreeMap::new(); + let mut get_child_exprs: BTreeMap<&node_types::TypeName, Vec>> = + BTreeMap::new(); + for (type_name, node) in nodes { + if let node_types::EntryKind::Table { + name: main_table_name, + fields, + } = &node.kind + { + if fields.is_empty() { + panic!("Encountered node '{}' with no fields", type_name.kind); + } + + // Count how many columns there will be in the main table. There + // will be one for the id, plus one for each field that's stored + // as a column. + let main_table_arity = 1 + fields + .iter() + .filter(|&f| matches!(f.storage, node_types::Storage::Column { .. })) + .count(); + + let mut main_table_column_index: usize = 0; + let mut predicates = Vec::new(); + let mut exprs = Vec::new(); + for field in fields { + let (get_preds, get_child_expr) = create_field_getters( + main_table_name, + main_table_arity, + &mut main_table_column_index, + field, + nodes, + ); + predicates.extend(get_preds); + if let Some(get_child_expr) = get_child_expr { + exprs.push(get_child_expr) + } + } + field_predicates.insert(type_name, predicates); + get_child_exprs.insert(type_name, exprs); + } + } + + // Next, for every supertype (union) node, compute the predicates that are + // guaranteed to be defined identically (in name, return type, and formal + // parameters) by every one of its members. Such predicates can be hoisted + // to an `abstract` predicate on the supertype's class, with the + // corresponding predicates on its members becoming `override`s. + let mut exposed_predicates: BTreeMap<&str, Vec>> = BTreeMap::new(); + for (type_name, node) in nodes { + if let node_types::EntryKind::Union { .. } = &node.kind { + compute_exposed_predicates( + type_name, + nodes, + &field_predicates, + &mut exposed_predicates, + ); + } + } + for (type_name, node) in nodes { match &node.kind { node_types::EntryKind::Token { kind_id: _ } => { @@ -865,7 +1016,26 @@ pub fn convert_nodes(nodes: &node_types::NodeTypeMap) -> Vec> { } node_types::EntryKind::Union { members: _ } => { // It's a tree-sitter supertype node, so we're wrapping a dbscheme - // union type. + // union type. Any predicate that's identically defined by every + // member becomes an `abstract` predicate here. + let predicates = exposed_predicates + .get(node.ql_class_name.as_str()) + .cloned() + .unwrap_or_default() + .into_iter() + .map(|predicate| ql::Predicate { + overridden: is_predicate_inherited( + &predicate, + type_name, + &direct_supertypes, + &exposed_predicates, + ), + is_private: false, + is_final: false, + body: None, + ..predicate + }) + .collect(); classes.push(ql::TopLevel::Class(ql::Class { qldoc: None, name: &node.ql_class_name, @@ -879,25 +1049,10 @@ pub fn convert_nodes(nodes: &node_types::NodeTypeMap) -> Vec> { &direct_supertypes, ), characteristic_predicate: None, - predicates: vec![], + predicates, })); } - node_types::EntryKind::Table { - name: main_table_name, - fields, - } => { - if fields.is_empty() { - panic!("Encountered node '{}' with no fields", type_name.kind); - } - - // Count how many columns there will be in the main table. There - // will be one for the id, plus one for each field that's stored - // as a column. - let main_table_arity = 1 + fields - .iter() - .filter(|&f| matches!(f.storage, node_types::Storage::Column { .. })) - .count(); - + node_types::EntryKind::Table { .. } => { let main_class_name = &node.ql_class_name; let mut main_class = ql::Class { qldoc: Some(format!("A class representing `{}` nodes.", type_name.kind)), @@ -915,26 +1070,30 @@ pub fn convert_nodes(nodes: &node_types::NodeTypeMap) -> Vec> { predicates: vec![create_get_a_primary_ql_class(main_class_name, true)], }; - let mut main_table_column_index: usize = 0; - let mut get_child_exprs: Vec = Vec::new(); - - // Iterate through the fields, creating: - // - classes to wrap union types if fields need them, - // - predicates to access the fields, - // - the QL expressions to access the fields that will be part of getAFieldOrChild. - for field in fields { - let (get_preds, get_child_expr) = create_field_getters( - main_table_name, - main_table_arity, - &mut main_table_column_index, - field, - nodes, - ); - main_class.predicates.extend(get_preds); - if let Some(get_child_expr) = get_child_expr { - get_child_exprs.push(get_child_expr) - } - } + // A field getter that's identically defined (in signature) by + // every member of one of this node's direct supertypes is an + // override of the corresponding `abstract` predicate declared + // there. + main_class.predicates.extend( + field_predicates + .get(type_name) + .cloned() + .unwrap_or_default() + .into_iter() + .map(|predicate| { + let overridden = predicate.overridden + || is_predicate_inherited( + &predicate, + type_name, + &direct_supertypes, + &exposed_predicates, + ); + ql::Predicate { + overridden, + ..predicate + } + }), + ); main_class.predicates.push(ql::Predicate { qldoc: Some(String::from("Gets a field or child node of this node.")), @@ -944,7 +1103,9 @@ pub fn convert_nodes(nodes: &node_types::NodeTypeMap) -> Vec> { is_final: true, return_type: Some(ql::Type::Facade("AstNode")), formal_parameters: vec![], - body: ql::Expression::Or(get_child_exprs), + body: Some(ql::Expression::Or( + get_child_exprs.get(type_name).cloned().unwrap_or_default(), + )), overlay: None, }); @@ -1038,7 +1199,7 @@ pub fn create_print_ast_module(nodes: &node_types::NodeTypeMap) -> ql::TopLevel< param_type: ql::Type::Int, }, ], - body: ql::Expression::Or(disjuncts), + body: Some(ql::Expression::Or(disjuncts)), overlay: None, }; From a96622e3feba400a5857480228d4ed696ceebdd4 Mon Sep 17 00:00:00 2001 From: Anders Schack-Mulligen Date: Fri, 4 Sep 2026 07:57:48 +0200 Subject: [PATCH 11/19] Unified: Regenerate Ast.qll --- .../ql/lib/codeql/unified/internal/Ast.qll | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/unified/ql/lib/codeql/unified/internal/Ast.qll b/unified/ql/lib/codeql/unified/internal/Ast.qll index cf927e257a9c..d47bd07d142e 100644 --- a/unified/ql/lib/codeql/unified/internal/Ast.qll +++ b/unified/ql/lib/codeql/unified/internal/Ast.qll @@ -104,7 +104,7 @@ module Unified { final F::AccessorKind getAccessorKind() { unified_accessor_declaration_def(this, result, _) } /** Gets the node corresponding to the field `body`. */ - final F::Block getBody() { unified_accessor_declaration_body(this, result) } + final override F::Block getBody() { unified_accessor_declaration_body(this, result) } /** Gets the node corresponding to the field `modifier`. */ final F::Modifier getModifier(int i) { unified_accessor_declaration_modifier(this, i, result) } @@ -373,7 +373,10 @@ module Unified { } } - class Callable extends @unified_callable, F::AstNode { } + class Callable extends @unified_callable, F::AstNode { + /** Gets the node corresponding to the field `body`. */ + abstract F::Block getBody(); + } /** A class representing `catch_clause` nodes. */ class CatchClause extends @unified_catch_clause, F::AstNode { @@ -512,7 +515,7 @@ module Unified { final override string getAPrimaryQlClass() { result = "ConstructorDeclaration" } /** Gets the node corresponding to the field `body`. */ - final F::Block getBody() { unified_constructor_declaration_def(this, result) } + final override F::Block getBody() { unified_constructor_declaration_def(this, result) } /** Gets the node corresponding to the field `modifier`. */ final F::Modifier getModifier(int i) { @@ -592,7 +595,7 @@ module Unified { final override string getAPrimaryQlClass() { result = "DestructorDeclaration" } /** Gets the node corresponding to the field `body`. */ - final F::Block getBody() { unified_destructor_declaration_def(this, result) } + final override F::Block getBody() { unified_destructor_declaration_def(this, result) } /** Gets the node corresponding to the field `modifier`. */ final F::Modifier getModifier(int i) { @@ -727,7 +730,7 @@ module Unified { final override string getAPrimaryQlClass() { result = "FunctionDeclaration" } /** Gets the node corresponding to the field `body`. */ - final F::Block getBody() { unified_function_declaration_body(this, result) } + final override F::Block getBody() { unified_function_declaration_body(this, result) } /** Gets the node corresponding to the field `modifier`. */ final F::Modifier getModifier(int i) { unified_function_declaration_modifier(this, i, result) } @@ -783,7 +786,7 @@ module Unified { final override string getAPrimaryQlClass() { result = "FunctionExpr" } /** Gets the node corresponding to the field `body`. */ - final F::Block getBody() { unified_function_expr_def(this, result) } + final override F::Block getBody() { unified_function_expr_def(this, result) } /** Gets the node corresponding to the field `capture_declaration`. */ final F::VariableDeclaration getCaptureDeclaration(int i) { @@ -958,7 +961,7 @@ module Unified { final override string getAPrimaryQlClass() { result = "InitializerDeclaration" } /** Gets the node corresponding to the field `body`. */ - final F::Block getBody() { unified_initializer_declaration_def(this, result) } + final override F::Block getBody() { unified_initializer_declaration_def(this, result) } /** Gets the node corresponding to the field `modifier`. */ final F::Modifier getModifier(int i) { @@ -1359,7 +1362,7 @@ module Unified { final override string getAPrimaryQlClass() { result = "TopLevel" } /** Gets the node corresponding to the field `body`. */ - final F::Block getBody() { unified_top_level_def(this, result) } + final override F::Block getBody() { unified_top_level_def(this, result) } /** Gets a field or child node of this node. */ final override F::AstNode getAFieldOrChild() { unified_top_level_def(this, result) } From 04865cbecad113e59fd3c86ae388d08195f75839 Mon Sep 17 00:00:00 2001 From: Anders Schack-Mulligen Date: Fri, 4 Sep 2026 07:58:47 +0200 Subject: [PATCH 12/19] Unified: Simplify callableGetBody. --- .../lib/codeql/unified/internal/ControlFlowGraph.qll | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/unified/ql/lib/codeql/unified/internal/ControlFlowGraph.qll b/unified/ql/lib/codeql/unified/internal/ControlFlowGraph.qll index 15db0eb3836e..47bda0e16dd3 100644 --- a/unified/ql/lib/codeql/unified/internal/ControlFlowGraph.qll +++ b/unified/ql/lib/codeql/unified/internal/ControlFlowGraph.qll @@ -46,15 +46,7 @@ private module Ast implements AstSig { class Callable = U::Callable; - AstNode callableGetBody(Callable c) { - result = c.(AccessorDeclaration).getBody() or - result = c.(ConstructorDeclaration).getBody() or - result = c.(DestructorDeclaration).getBody() or - result = c.(FunctionDeclaration).getBody() or - result = c.(FunctionExpr).getBody() or - result = c.(InitializerDeclaration).getBody() or - result = c.(TopLevel).getBody() - } + AstNode callableGetBody(Callable c) { result = c.getBody() } class Parameter extends U::Parameter { Expr getDefaultValue() { result = super.getDefault() } From d456656e6e4bec3ccf547ede8aa006a01fd5b85a Mon Sep 17 00:00:00 2001 From: Anders Schack-Mulligen Date: Fri, 4 Sep 2026 10:28:02 +0200 Subject: [PATCH 13/19] Guard against infinite recursion on malformed supertype declarations. --- shared/tree-sitter-extractor/src/generator/ql_gen.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/shared/tree-sitter-extractor/src/generator/ql_gen.rs b/shared/tree-sitter-extractor/src/generator/ql_gen.rs index 04a9e73aee1b..b6f3d45f4b12 100644 --- a/shared/tree-sitter-extractor/src/generator/ql_gen.rs +++ b/shared/tree-sitter-extractor/src/generator/ql_gen.rs @@ -862,6 +862,9 @@ fn compute_exposed_predicates<'a, 'b>( let node = nodes.get(type_name); let class_name = node.map_or(type_name.kind.as_str(), |node| node.ql_class_name.as_str()); if !cache.contains_key(class_name) { + // Supertype declarations that recursively refer to themselves are a mistake, but we don't + // want to cause infinite recursion, so we insert a temporary sentinel. + cache.insert(class_name, Vec::new()); let exposed = match node.map(|node| &node.kind) { Some(node_types::EntryKind::Table { .. }) => { field_predicates.get(type_name).cloned().unwrap_or_default() From 0565ba384b63b743d2edee165c5aa75102ea6d55 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Fri, 4 Sep 2026 11:03:40 +0200 Subject: [PATCH 14/19] C++: Update supported compiler extensions in docs --- docs/codeql/reusables/supported-versions-compilers.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/codeql/reusables/supported-versions-compilers.rst b/docs/codeql/reusables/supported-versions-compilers.rst index 8651fa91a269..e341918e5931 100644 --- a/docs/codeql/reusables/supported-versions-compilers.rst +++ b/docs/codeql/reusables/supported-versions-compilers.rst @@ -4,9 +4,9 @@ :stub-columns: 1 Language,Variants,Compilers,Extensions - C/C++,"C89, C99, C11, C17, C23, C++98, C++03, C++11, C++14, C++17, C++20, C++23 [1]_ [2]_ [3]_","Clang (including clang-cl and armclang) extensions (up to Clang 21), + C/C++,"C89, C99, C11, C17, C23, C++98, C++03, C++11, C++14, C++17, C++20, C++23 [1]_ [2]_ [3]_","Clang (including clang-cl and armclang) extensions (up to Clang 22), - GNU extensions (up to GCC 15), + GNU extensions (up to GCC 16), Microsoft extensions (up to VS 2022), From 56f5e447ce588328babe14b9798af9e4b76f02ce Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Fri, 4 Sep 2026 11:37:32 +0200 Subject: [PATCH 15/19] Kotlin: fix broken formatting --- .../src/main/java/com/semmle/util/files/FileUtil.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/kotlin-extractor/src/main/java/com/semmle/util/files/FileUtil.java b/java/kotlin-extractor/src/main/java/com/semmle/util/files/FileUtil.java index 19bdc786c5e2..16df41e5341e 100644 --- a/java/kotlin-extractor/src/main/java/com/semmle/util/files/FileUtil.java +++ b/java/kotlin-extractor/src/main/java/com/semmle/util/files/FileUtil.java @@ -1244,8 +1244,8 @@ public static File tryMakeCanonical (File f) try { // getCanonicalFile does not canonicalize subst drives on Windows, so do this separately. This // is a no-op on non-Windows platforms. - return SubstResolver.resolve(f.getCanonicalFile()); } - catch (IOException ignored) { + return SubstResolver.resolve(f.getCanonicalFile()); + } catch (IOException ignored) { Exceptions.ignore(ignored, "Can't log error: Could be too verbose."); return new File(simplifyPath(f)); } From 65615a79a81ebde5a2a7682bb3e6350491343925 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Fri, 4 Sep 2026 13:34:55 +0100 Subject: [PATCH 16/19] C++: Fix join in virtual dispatch's 'returnStep' predicate. --- .../ir/dataflow/internal/DataFlowDispatch.qll | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll index bce936552768..03a565ef946d 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowDispatch.qll @@ -131,6 +131,15 @@ private predicate qualifierSourceImpl(RelevantNode n, Class c) { ) } +pragma[nomagic] +private predicate hasKindAndEnclosingCallable( + DataFlowPrivate::DataFlowCallable callable, DataFlowPrivate::ReturnKind kind, + DataFlowPrivate::ReturnNode return +) { + return.getEnclosingCallable() = callable and + return.getKind() = kind +} + private module TrackVirtualDispatch { /** * Gets a possible runtime target of `c` using both static call-target @@ -197,11 +206,21 @@ private module TrackVirtualDispatch { ) } + pragma[nomagic] + private predicate hasDispatchWithKind( + DataFlowPrivate::DataFlowCallable callable, DataFlowPrivate::ReturnKind kind, + LocalSourceNode n2 + ) { + exists(DataFlowPrivate::DataFlowCall call | + n2 = DataFlowPrivate::getAnOutNode(call, kind) and + callable = dispatch(call) + ) + } + predicate returnStep(Node n1, LocalSourceNode n2) { - exists(DataFlowPrivate::DataFlowCallable callable, DataFlowPrivate::DataFlowCall call | - n1.(DataFlowPrivate::ReturnNode).getEnclosingCallable() = callable and - callable = dispatch(call) and - n2 = DataFlowPrivate::getAnOutNode(call, n1.(DataFlowPrivate::ReturnNode).getKind()) + exists(DataFlowPrivate::DataFlowCallable callable, DataFlowPrivate::ReturnKind kind | + hasKindAndEnclosingCallable(callable, kind, n1) and + hasDispatchWithKind(callable, kind, n2) ) } From b42f997f1e7d1101f4ef66479767b2d3d0ff8874 Mon Sep 17 00:00:00 2001 From: Jeroen Ketema Date: Fri, 4 Sep 2026 17:00:48 +0200 Subject: [PATCH 17/19] C++: Fix test I broke by accident --- .../taint-tests/test_mad-signatures.expected | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cpp/ql/test/library-tests/dataflow/taint-tests/test_mad-signatures.expected b/cpp/ql/test/library-tests/dataflow/taint-tests/test_mad-signatures.expected index d494c09e71d5..2da7e83cca37 100644 --- a/cpp/ql/test/library-tests/dataflow/taint-tests/test_mad-signatures.expected +++ b/cpp/ql/test/library-tests/dataflow/taint-tests/test_mad-signatures.expected @@ -1962,6 +1962,15 @@ getSignatureParameterName | (BUF_MEM *,size_t) | | BUF_MEM_grow | 1 | size_t | | (BUF_MEM *,size_t) | | BUF_MEM_grow_clean | 0 | BUF_MEM * | | (BUF_MEM *,size_t) | | BUF_MEM_grow_clean | 1 | size_t | +| (Blob *,int,const Blob &,int,int) | BlobUtil | copy | 0 | Blob * | +| (Blob *,int,const Blob &,int,int) | BlobUtil | copy | 1 | int | +| (Blob *,int,const Blob &,int,int) | BlobUtil | copy | 2 | const Blob & | +| (Blob *,int,const Blob &,int,int) | BlobUtil | copy | 3 | int | +| (Blob *,int,const Blob &,int,int) | BlobUtil | copy | 4 | int | +| (Blob *,int,const char *,int) | BlobUtil | copy | 0 | Blob * | +| (Blob *,int,const char *,int) | BlobUtil | copy | 1 | int | +| (Blob *,int,const char *,int) | BlobUtil | copy | 2 | const char * | +| (Blob *,int,const char *,int) | BlobUtil | copy | 3 | int | | (BrotliBitReader *const,uint64_t,uint64_t *) | | BrotliSafeReadBits32Slow | 0 | BrotliBitReader *const | | (BrotliBitReader *const,uint64_t,uint64_t *) | | BrotliSafeReadBits32Slow | 1 | uint64_t | | (BrotliBitReader *const,uint64_t,uint64_t *) | | BrotliSafeReadBits32Slow | 2 | uint64_t * | @@ -13127,6 +13136,10 @@ getSignatureParameterName | (char *,char,char **) | | __old_strtok_r_1c | 0 | char * | | (char *,char,char **) | | __old_strtok_r_1c | 1 | char | | (char *,char,char **) | | __old_strtok_r_1c | 2 | char ** | +| (char *,const Blob &,int,int) | BlobUtil | copy | 0 | char * | +| (char *,const Blob &,int,int) | BlobUtil | copy | 1 | const Blob & | +| (char *,const Blob &,int,int) | BlobUtil | copy | 2 | int | +| (char *,const Blob &,int,int) | BlobUtil | copy | 3 | int | | (char *,const char *) | | xstrdup | 0 | char * | | (char *,const char *) | | xstrdup | 1 | const char * | | (char *,const char **,const char **,const char **,const char **,const char **) | | _nl_explode_name | 0 | char * | From 8bc69b4012c0cc3515e4a55143710ffee34c7c37 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Fri, 4 Sep 2026 17:04:21 +0100 Subject: [PATCH 18/19] C++: Fix a bad join. Before: ``` [2026-09-04 12:36:27] Evaluated non-recursive predicate FlowSummaryImpl::Input2::hasKindAndEnclosingFunction/3#3e350d0c@603f9du6 in 374ms (size: 604170). Evaluated relational algebra for predicate FlowSummaryImpl::Input2::hasKindAndEnclosingFunction/3#3e350d0c@603f9du6 with tuple counts: 672310 ~0% {2} r1 = JOIN `cached_ResolveFunction::isFunction/1#9226b83f` WITH DataFlowPrivate::TSourceCallable#54d42094 ON FIRST 1 OUTPUT Rhs.1, Lhs.0 34670069 ~2% {2} | JOIN WITH `DataFlowUtil::Node.getEnclosingCallable/0#dispred#74002437_10#join_rhs` ON FIRST 1 OUTPUT Rhs.1, Lhs.1 604170 ~0% {3} | JOIN WITH `DataFlowPrivate::ReturnNode.getKind/0#dispred#c7586c0b` ON FIRST 1 OUTPUT Lhs.1, Rhs.1, Lhs.0 return r1 ``` After: ``` [2026-09-04 16:46:20] Evaluated non-recursive predicate FlowSummaryImpl::Input2::hasKindAndEnclosingFunction/3#3e350d0c@1ded09ff in 1158ms (size: 604170). Evaluated relational algebra for predicate FlowSummaryImpl::Input2::hasKindAndEnclosingFunction/3#3e350d0c@1ded09ff with tuple counts: 614924 ~1% {3} r1 = JOIN `DataFlowPrivate::ReturnNode.getKind/0#dispred#c7586c0b` WITH `DataFlowUtil::Node.getEnclosingCallable/0#dispred#74002437` ON FIRST 1 OUTPUT Rhs.1, Lhs.1, Lhs.0 604170 ~5% {3} | JOIN WITH DataFlowPrivate::TSourceCallable#54d42094_10#join_rhs ON FIRST 1 OUTPUT Rhs.1, Lhs.1, Lhs.2 604170 ~5% {3} | JOIN WITH `cached_ResolveFunction::isFunction/1#9226b83f` ON FIRST 1 OUTPUT Lhs.0, Lhs.1, Lhs.2 return r1 ``` --- .../lib/semmle/code/cpp/dataflow/internal/FlowSummaryImpl.qll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/FlowSummaryImpl.qll b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/FlowSummaryImpl.qll index 780c802dc8ae..176b95933db8 100644 --- a/cpp/ql/lib/semmle/code/cpp/dataflow/internal/FlowSummaryImpl.qll +++ b/cpp/ql/lib/semmle/code/cpp/dataflow/internal/FlowSummaryImpl.qll @@ -256,7 +256,7 @@ private module Input2 implements Impl::Private::InputSig2 { pragma[nomagic] private predicate hasKindAndEnclosingFunction(Function f, ReturnKind rk, ReturnNode r) { r.getEnclosingCallable().asSourceCallable() = f and - r.getKind() = rk + pragma[only_bind_into](r).getKind() = rk } pragma[nomagic] From 0208f19c639b25315b61fd250a5184cb822325b7 Mon Sep 17 00:00:00 2001 From: Mathias Vorreiter Pedersen Date: Fri, 4 Sep 2026 17:05:14 +0100 Subject: [PATCH 19/19] C++: Reduce tuple duplication in store step pipeline. Before (notice the large tuple duplication): ``` [2026-09-04 12:36:27] Evaluated non-recursive predicate DataFlowPrivate::storeStepImpl/4#5a9e2fd2@8e5724c9 in 1535ms (size: 189539). Evaluated relational algebra for predicate DataFlowPrivate::storeStepImpl/4#5a9e2fd2@8e5724c9 with tuple counts: 20361 ~4% {3} r1 = JOIN DataFlowNodes::TFlowSummaryNode#d5706fd6 WITH `FlowSummaryImpl::Private::Steps::summaryStoreStep/3#a7d89e4d` ON FIRST 1 OUTPUT Rhs.2, Lhs.1, Rhs.1 20361 ~0% {4} | JOIN WITH DataFlowNodes::TFlowSummaryNode#d5706fd6 ON FIRST 1 OUTPUT Lhs.1, Lhs.2, Rhs.1, _ 20361 ~0% {4} | REWRITE WITH Out.3 := true 27390490 ~0% {3} r2 = SCAN `DataFlowPrivate::nodeHasInstruction/3#f469bb06` OUTPUT In.1, In.0, In.2 1146769 ~1% {3} | JOIN WITH `Instruction::StoreInstruction.getDestinationAddressOperand/0#dispred#596a4aba` ON FIRST 1 OUTPUT Rhs.1, Lhs.1, Lhs.2 1251911 ~0% {6} | JOIN WITH `DataFlowPrivate::numberOfLoadsFromOperand/4#7e555666_1023#join_rhs` ON FIRST 1 OUTPUT Rhs.1, Lhs.1, Rhs.3, _, Lhs.2, Rhs.2 1251911 ~3% {4} | REWRITE WITH Tmp.3 := 1, Out.3 := (Tmp.3 + In.4 + In.5) KEEPING 4 354360 ~0% {6} | JOIN WITH DataFlowNodes::PostFieldUpdateNode#ba49e082_1023#join_rhs ON FIRST 1 OUTPUT Rhs.1, Lhs.1, Lhs.2, Lhs.3, Rhs.2, Rhs.3 354360 ~0% {8} | JOIN WITH DataFlowNodes::TPostUpdateNodeImpl#15a1088b_21#join_rhs ON FIRST 1 OUTPUT Lhs.1, Lhs.2, Lhs.3, Lhs.0, Lhs.4, Lhs.5, Rhs.1, _ {7} | REWRITE WITH Tmp.7 := 1, TEST InOut.6 = Tmp.7 KEEPING 7 170359 ~1% {6} | SCAN OUTPUT In.3, In.4, In.5, In.0, In.1, In.2 697388 ~3% {5} | JOIN WITH DataFlowNodes::PostFieldUpdateNode#ba49e082_0231#join_rhs ON FIRST 3 OUTPUT Rhs.3, Lhs.3, Lhs.4, Lhs.5, Lhs.0 697555 ~302% {5} | JOIN WITH `DataFlowNodes::FieldAddress.getField/0#dispred#fea3b845` ON FIRST 1 OUTPUT Rhs.1, Lhs.1, Lhs.2, Lhs.3, Lhs.4 895112 ~156% {5} | JOIN WITH `DataFlowUtil::FieldContent.getAField/0#dispred#ba1c91e5_10#join_rhs` ON FIRST 1 OUTPUT Rhs.1, Lhs.3, Lhs.1, Lhs.2, Lhs.4 417949 ~139% {4} | JOIN WITH `DataFlowUtil::Content.getIndirectionIndex/0#dispred#c14b335b` ON FIRST 2 OUTPUT Lhs.2, Lhs.0, Lhs.4, Lhs.3 438310 ~121% {4} r3 = r1 UNION r2 return r3 ``` After: ``` [2026-09-04 16:30:27] Evaluated non-recursive predicate DataFlowPrivate::storeStepTarget/4#0e049f2e@635fa9rj in 72ms (size: 699622). Evaluated relational algebra for predicate DataFlowPrivate::storeStepTarget/4#0e049f2e@635fa9rj with tuple counts: 702928 ~0% {4} r1 = JOIN `DataFlowPrivate::hasFieldAndIndirectionIndex/3#a1231a07` WITH `DataFlowPrivate::hasFieldAddressAndField/3#35a98069` ON FIRST 1 OUTPUT Rhs.2, Lhs.1, Rhs.1, Lhs.2 return r1 [2026-09-04 16:30:38] Evaluated non-recursive predicate DataFlowPrivate::storeStepSource/4#765254a7@d27b3d3c in 163ms (size: 1251911). Evaluated relational algebra for predicate DataFlowPrivate::storeStepSource/4#765254a7@d27b3d3c with tuple counts: 863285 ~0% {2} r1 = JOIN `Instruction::StoreInstruction.getDestinationAddressOperand/0#dispred#596a4aba` WITH Instruction::StoreInstruction#ae96f30c ON FIRST 1 OUTPUT Lhs.0, Lhs.1 1146769 ~2% {3} | JOIN WITH `DataFlowPrivate::nodeHasInstruction/3#f469bb06_102#join_rhs` ON FIRST 1 OUTPUT Lhs.1, Rhs.1, Rhs.2 1251911 ~1% {6} | JOIN WITH `DataFlowPrivate::numberOfLoadsFromOperand/4#7e555666_1023#join_rhs` ON FIRST 1 OUTPUT Rhs.1, _, Lhs.1, Rhs.3, Lhs.2, Rhs.2 1251911 ~1% {4} | REWRITE WITH Tmp.1 := 1, Out.1 := (Tmp.1 + In.4 + In.5) KEEPING 4 return r1 [2026-09-04 16:30:38] Evaluated non-recursive predicate DataFlowPrivate::storeStepImpl/4#5a9e2fd2@ddd0439f in 70ms (size: 189539). Evaluated relational algebra for predicate DataFlowPrivate::storeStepImpl/4#5a9e2fd2@ddd0439f with tuple counts: 169178 ~0% {4} r1 = JOIN `DataFlowPrivate::storeStepTarget/4#0e049f2e` WITH `DataFlowPrivate::storeStepSource/4#765254a7` ON FIRST 2 OUTPUT Rhs.2, Lhs.3, Lhs.2, Rhs.3 20361 ~4% {3} r2 = JOIN DataFlowNodes::TFlowSummaryNode#d5706fd6 WITH `FlowSummaryImpl::Private::Steps::summaryStoreStep/3#a7d89e4d` ON FIRST 1 OUTPUT Rhs.2, Lhs.1, Rhs.1 20361 ~0% {4} | JOIN WITH DataFlowNodes::TFlowSummaryNode#d5706fd6 ON FIRST 1 OUTPUT Lhs.1, Lhs.2, Rhs.1, _ 20361 ~0% {4} | REWRITE WITH Out.3 := true 189539 ~0% {4} r3 = r1 UNION r2 return r3 ``` --- .../ir/dataflow/internal/DataFlowPrivate.qll | 57 ++++++++++++++----- 1 file changed, 44 insertions(+), 13 deletions(-) diff --git a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll index 3a1b42645642..551035c5589e 100644 --- a/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll +++ b/cpp/ql/lib/semmle/code/cpp/ir/dataflow/internal/DataFlowPrivate.qll @@ -122,6 +122,47 @@ private module Cached { FlowSummaryImpl::Private::Steps::summaryJumpStep(n1, n2) } + bindingset[store] + pragma[inline_late] + private predicate nodeHasInstructionLate(Node node, StoreInstruction store, int indirectionIndex) { + nodeHasInstruction(node, store, indirectionIndex) + } + + pragma[nomagic] + private predicate storeStepSource( + Operand fieldAddress, int contentIndirectionIndex, Node node, boolean certain + ) { + exists(int indirectionIndex, int numberOfLoads, StoreInstruction store | + nodeHasInstructionLate(node, store, indirectionIndex) and + numberOfLoadsFromOperand(fieldAddress, store.getDestinationAddressOperand(), numberOfLoads, + certain) and + contentIndirectionIndex = 1 + indirectionIndex + numberOfLoads + ) + } + + pragma[nomagic] + private predicate hasFieldAddressAndField(Field f, PostFieldUpdateNode pfu, Operand fieldAddress) { + pfu.getIndirectionIndex() = 1 and + pfu.getUpdatedField() = f and + pfu.getFieldAddress() = fieldAddress + } + + pragma[nomagic] + private predicate hasFieldAndIndirectionIndex(Field f, int indirectionIndex, FieldContent fc) { + fc.getAField() = f and + fc.getIndirectionIndex() = indirectionIndex + } + + pragma[nomagic] + private predicate storeStepTarget( + Operand address, int indirectionIndex, PostFieldUpdateNode pfu, FieldContent fc + ) { + exists(Field f | + hasFieldAddressAndField(f, pfu, address) and + hasFieldAndIndirectionIndex(f, indirectionIndex, fc) + ) + } + /** * Holds if data can flow from `node1` to `node2` via an assignment to `f`. * Thus, `node2` references an object with a field `f` that contains the @@ -132,19 +173,9 @@ private module Cached { */ cached predicate storeStepImpl(Node node1, Content c, Node node2, boolean certain) { - exists( - PostFieldUpdateNode postFieldUpdate, int indirectionIndex1, int numberOfLoads, - StoreInstruction store, FieldContent fc - | - postFieldUpdate = node2 and - fc = c and - nodeHasInstruction(node1, pragma[only_bind_into](store), - pragma[only_bind_into](indirectionIndex1)) and - postFieldUpdate.getIndirectionIndex() = 1 and - numberOfLoadsFromOperand(postFieldUpdate.getFieldAddress(), - store.getDestinationAddressOperand(), numberOfLoads, certain) and - fc.getAField() = postFieldUpdate.getUpdatedField() and - getIndirectionIndexLate(fc) = 1 + indirectionIndex1 + numberOfLoads + exists(Operand fieldAddress, int indirectionIndex | + storeStepSource(fieldAddress, indirectionIndex, node1, certain) and + storeStepTarget(fieldAddress, indirectionIndex, node2, c) ) or // models-as-data summarized flow