diff --git a/.github/workflows/compute-entropy.py b/.github/workflows/compute-entropy.py index 42655913bb..44a11770a7 100644 --- a/.github/workflows/compute-entropy.py +++ b/.github/workflows/compute-entropy.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 -import sys, math, json, subprocess +import os +import sys +import math +import json +import subprocess from collections import Counter from pathlib import Path @@ -10,23 +14,36 @@ def shannon_entropy(text: str) -> float: probs = [count / len(text) for count in freq.values()] return -sum(p * math.log2(p) for p in probs if p > 0) -# Get changed files safely for pull_request events +# Determine changed files based on event type changed_files = [] +event_name = os.environ.get("EVENT_NAME", "") +base_sha = os.environ.get("BASE_SHA", "").strip() +head_sha = os.environ.get("HEAD_SHA", "").strip() or "HEAD" + try: - # GitHub provides github.event.pull_request.base.sha and head.sha in the context - base_sha = subprocess.check_output(['git', 'rev-parse', 'origin/${{ github.base_ref }}'], text=True).strip() - changed_files = subprocess.check_output( - ['git', 'diff', '--name-only', base_sha, 'HEAD'], text=True - ).splitlines() + if event_name == "pull_request_target" and base_sha: + # PR case: we checked out the PR head and fetched the base commit + changed_files = subprocess.check_output( + ["git", "diff", "--name-only", base_sha, head_sha], + text=True + ).splitlines() + else: + # push / release: diff against previous commit + changed_files = subprocess.check_output( + ["git", "diff", "--name-only", "HEAD~1", "HEAD"], + text=True + ).splitlines() except subprocess.CalledProcessError: - # Fallback for first-time PRs or edge cases: use the merge-base or just files in HEAD + # Fallbacks try: changed_files = subprocess.check_output( - ['git', 'diff', '--name-only', 'HEAD~1', 'HEAD'], text=True + ["git", "diff", "--name-only", "HEAD~1", "HEAD"], + text=True ).splitlines() except subprocess.CalledProcessError: - # Last resort: all files in the repo - changed_files = subprocess.check_output(['git', 'ls-files'], text=True).splitlines() + changed_files = subprocess.check_output( + ["git", "ls-files"], text=True + ).splitlines() results = [] total_ent = 0.0 @@ -37,7 +54,7 @@ def shannon_entropy(text: str) -> float: if not path.exists() or path.suffix in {'.png', '.jpg', '.gif', '.bin', '.lock', '.exe', '.dll', '.so'}: continue try: - content = path.read_text(encoding='utf-8', errors='ignore') + content = path.read_text(encoding="utf-8", errors="ignore") ent = shannon_entropy(content) results.append(f"{f}: {ent:.3f}") total_ent += ent @@ -53,7 +70,7 @@ def shannon_entropy(text: str) -> float: "No source files changed" ) -with open('/tmp/beauty.json', 'w') as f: +with open("/tmp/beauty.json", "w") as f: json.dump({ "average_entropy": avg, "verdict": verdict, @@ -61,4 +78,4 @@ def shannon_entropy(text: str) -> float: }, f, indent=2) print(f"Average entropy: {avg}") -print(verdict) +print(verdict) \ No newline at end of file diff --git a/.github/workflows/entropy-beauty-scan.yml b/.github/workflows/entropy-beauty-scan.yml index b95588696d..4c0c915731 100644 --- a/.github/workflows/entropy-beauty-scan.yml +++ b/.github/workflows/entropy-beauty-scan.yml @@ -11,12 +11,15 @@ jobs: scan: runs-on: ubuntu-latest steps: - - name: Checkout code (full history) - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: Checkout code + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # pinned commit (update sha if you bump the action) with: - # Use shallow clone by default (much faster) - # Only use full history when necessary (push/release or when base commit is needed) - fetch-depth: ${{ github.event_name == 'pull_request_target' && 2 || 0 }} + ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || github.sha }} + fetch-depth: ${{ github.event_name == 'pull_request_target' && 1 || 2 }} + + - name: Fetch PR base commit (needed for accurate diff) + if: github.event_name == 'pull_request_target' + run: git fetch origin ${{ github.event.pull_request.base.sha }} --depth=1 - name: Cache pip manually uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -43,6 +46,10 @@ jobs: extra_args: --results=verified,unknown --filter-entropy=3.5 --json - name: Compute mid-4 beauty entropy + env: + EVENT_NAME: ${{ github.event_name }} + BASE_SHA: ${{ github.event.pull_request.base.sha || '' }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} run: python .github/workflows/compute-entropy.py - name: Post summary comment (PR only) @@ -63,7 +70,6 @@ jobs: }).filter(Boolean); } catch(e) {} } else { - // Fallback: the action also logs to GITHUB_STEP_SUMMARY, but we use the file from the Python step console.log("No trufflehog.json found, using empty findings"); } @@ -91,6 +97,7 @@ jobs: issue_number: context.issue.number, body: body }); + # ── Create issue on push ONLY if suspicious (entropy outside 4.3–5.1) ── - name: Create issue on suspicious push if: github.event_name == 'push' || github.event_name == 'release' @@ -140,4 +147,4 @@ jobs: labels: ['entropy', 'security', 'review-needed'] }); - console.log("⚠️ Created issue because entropy was outside mid-4 range"); + console.log("⚠️ Created issue because entropy was outside mid-4 range"); \ No newline at end of file diff --git a/README.md b/README.md index d3a945fade..724fc3c7d7 100644 --- a/README.md +++ b/README.md @@ -21,13 +21,13 @@ It extends the [observer pattern](http://en.wikipedia.org/wiki/Observer_pattern) - :+1: New `Streamable` built around Virtual Threads & virtual blocking. Think `IAsyncEnumerable` for Java. :satellite: in progress. - :+1: Using Java Cleaner API to detect resource leaks and using it for adaptive cleanups. - :information_source: Reactive Streams Test Compatibility Kit usage; [Reactive-Streams](https://github.com/reactive-streams/reactive-streams-jvm). -- :satellite: Rewamp of the Javadoc bloat in the base types via `sealed` interfaces. - :satellite: Reduce overload bloat by using `record`-based configurations. - :satellite: Internal optimizations now that I have the master :key:. - :eye: Possible usages for Scoped variables for context and per-item resource management. - :eye: Possible inclusion of 2nd and 3rd party operators. - :eye: Possible inclusion of the Iterable Extensions (Ix) 2nd party library. ju.Stream is sh|t wrt interfacing and composability. - :question: Android compatibility depends on your API level and what desugaring is available. +- :question: Rewamp of the Javadoc bloat in the base types via `sealed` interfaces? Not certain how much it helps. - :lady_beetle: Resolve many anomalies and bugs with operators such as `groupBy`, `window`, `concat`, etc. - :warning: RxJava 3.x support will be toned down in the coming months, will be offered for +1 year after 4.x official release. @@ -106,6 +106,10 @@ When the dataflow runs through asynchronous steps, each step may perform differe In RxJava, the dedicated `Flowable` class is designated to support backpressure and `Observable` is dedicated to the non-backpressured operations (short sequences, GUI interactions, etc.). The other types, `Single`, `Maybe` and `Completable` don't support backpressure nor should they; there is always room to store one item temporarily. +Since 4.0.0, the `Streamable` type gives natural backpressure because producers and consumers have to wait for +each other to hand over data. Since waiting is blocking, the type natively works with Virtual Threaded `ExecutorService`s +and the new `Schedulers.virtual()` `Scheduler`. + #### Assembly time The preparation of dataflows by applying various intermediate operators happens in the so-called **assembly time**: @@ -192,9 +196,16 @@ Typically, you can move computations or blocking IO to some other thread via `su RxJava operators don't work with `Thread`s or `ExecutorService`s directly but with so-called `Scheduler`s that abstract away sources of concurrency behind a uniform API. RxJava 4 features several standard schedulers accessible via `Schedulers` utility class. - `Schedulers.computation()`: Run computation intensive work on a fixed number of dedicated threads in the background. Most asynchronous operators use this as their default `Scheduler`. -- `Schedulers.io()`: Run I/O-like or blocking operations on a dynamically changing set of threads. +- `Schedulers.cached()`: Run I/O-like or blocking operations on a dynamically changing set of threads backed by native OS threads. :warning: Can exhaust system resources! +- `Schedulers.virtual()`: Run I/O-like or blocking scatter-gather operations in a sequential manner on threads with virtualized stacks attached and detached to native OS threads on demand. :information_source: Helps with the issues around unboundedness of `Schedulers.cached()`. - `Schedulers.single()`: Run work on a single thread in a sequential and FIFO manner. - `Schedulers.trampoline()`: Run work in a sequential and FIFO manner in one of the participating threads, usually for testing purposes. +- `Schedulers.createParallel()`: Allows creating a `Scheduler` with an user-configurable worker pool size and other parameters to contrast `computation()` which is always set to `availableProcessors()`/configured amount globally. +- `Schedulers.createBlocking()`: Allows creating an event-loop style `Scheduler` which runs tasks and blocks on the thread calling `execute()`. Can be used to pull tasks onto a specific thread or have it itself run in a virtual threaded executor for maximum efficiency. +- `Scheduler.shared()`: Every `Scheduler` or `Worker` can now be shared and act like its own full `Scheduler` with lifecycle tracking and dispose support. I.e., share one worker of `cached()` like it is some kind of `Schedulers.single()`. + +In 4.x, the traditional I/O scheduler `Schedulers.io()` has been API deprecated and delegates to `Schedulers.cached()` for compatibility reasons. It is recommended you decide at these deprecated code locations which standard (or custom) scheduler +to use: `cached()` like before or the new `virtual()` for more efficient system resource usages. These are available on all JVM platforms but some specific platforms, such as Android, have their own typical `Scheduler`s defined: `AndroidSchedulers.mainThread()`, `SwingScheduler.instance()` or `JavaFXScheduler.platform()`. @@ -359,13 +370,14 @@ In such situations, there are usually two options to fix the transformation: 1) Each reactive base class features operators that can perform such conversions, including the protocol conversions, to match some other type. The following matrix shows the available conversion options: -| | Flowable | Observable | Single | Maybe | Completable | -|----------|----------|------------|--------|-------|-------------| -|**Flowable** | | `toObservable` | `first`, `firstOrError`, `single`, `singleOrError`, `last`, `lastOrError`1 | `firstElement`, `singleElement`, `lastElement` | `ignoreElements` | -|**Observable**| `toFlowable`2 | | `first`, `firstOrError`, `single`, `singleOrError`, `last`, `lastOrError`1 | `firstElement`, `singleElement`, `lastElement` | `ignoreElements` | -|**Single** | `toFlowable`3 | `toObservable` | | `toMaybe` | `ignoreElement` | -|**Maybe** | `toFlowable`3 | `toObservable` | `toSingle` | | `ignoreElement` | -|**Completable** | `toFlowable` | `toObservable` | `toSingle` | `toMaybe` | | +| | Flowable | Observable | Single | Maybe | Completable | Streamable | +|----------|----------|------------|--------|-------|-------------|------------| +|**Flowable** | | `toObservable` | `first`, `firstOrError`, `single`, `singleOrError`, `last`, `lastOrError`1 | `firstElement`, `singleElement`, `lastElement` | `ignoreElements` | `toStreamable` | +|**Observable**| `toFlowable`2 | | `first`, `firstOrError`, `single`, `singleOrError`, `last`, `lastOrError`1 | `firstElement`, `singleElement`, `lastElement` | `ignoreElements` | `toStreamable` | +|**Single** | `toFlowable`3 | `toObservable` | | `toMaybe` | `ignoreElement` | `toStreamable` | +|**Maybe** | `toFlowable`3 | `toObservable` | `toSingle` | | `ignoreElement` | `toStreamable` | +|**Completable** | `toFlowable` | `toObservable` | `toSingle` | `toMaybe` | | `toStreamable` | +|**Streamable** | `toFlowable` | `toObservable` | TBD | TBD | TBD | | 1: When turning a multivalued source into a single-valued source, one should decide which of the many source values should be considered as the result. @@ -446,18 +458,33 @@ This can get also ambiguous when functional interface types get involved as the #### Error handling -Dataflows can fail, at which point the error is emitted to the consumer(s). Sometimes though, multiple sources may fail at which point there is a choice whether or not wait for all of them to complete or fail. To indicate this opportunity, many operator names are suffixed with the `DelayError` words (while others feature a `delayError` or `delayErrors` boolean flag in one of their overloads): +Dataflows can fail, at which point the error is emitted to the consumer(s). Sometimes though, multiple sources may fail at which point there is a choice whether or not wait for all of them to complete or fail. + +:warning: With 4.x, the number of overloads were reduced and all `DelayError` methods have been folded into +configuration record parameter(s). + +To indicate this opportunity, operators now use configuration records, such as `StandardBufferedConfig`, `StandardConcurrentConfig` and `StandardConcurrentBufferedConfig` to pass along an error handling settings: ```java Flowable concat(Publisher> sources); -Flowable concatDelayError(Publisher> sources); +Flowable concat(Publisher> sources, StandardBufferedConfig config); + +var config = new StandardBufferedConfig(ErrorMode.BOUNDARY); + +var config = new StandardBufferedConfig(true); // equivalent to ErrorMode.END ``` -Of course, suffixes of various kinds may appear together: +The `ErrorMode` enum has 3 modes: + +- `IMMEDIATE` - emit the error as soon as possible +- `BOUNDARY` - wait until a boundary or inner source change, may not make sense for certain operators and will act as `END` +- `END` - wait until all sources (outer, inner) have terminated and present an aggregated error (usually via `CompositeException`) to the consumer. + +With varargs-style operators, the configuration record has to precede the varargs parameter: ```java -Flowable concatArrayEagerDelayError(Publisher... sources); +Flowable concatArrayEager(StandardConcurrentBufferedConfig config, Publisher... sources); ``` #### Base class vs base type @@ -466,16 +493,19 @@ The base classes can be considered heavy due to the sheer number of static and i | Type | Class | Interface | Consumer | |------|-------|-----------|----------| -| 0..N backpressured | `Flowable` | `Publisher`1 | `Subscriber` | +| 0..N backpressured | `Flowable` | `Flow.Publisher`1 | `Flow.Subscriber` | | 0..N unbounded | `Observable` | `ObservableSource`2 | `Observer` | | 1 element or error | `Single` | `SingleSource` | `SingleObserver` | | 0..1 element or error | `Maybe` | `MaybeSource` | `MaybeObserver` | | 0 element or error | `Completable` | `CompletableSource` | `CompletableObserver` | +| 0..N coordinated | `Streamable` | 3 | `Streamer`, `StreamSink` | 1The `java.util.concurrent.Flow.Publisher` is part of the Java internal Flow library. It is the main type to interact with other reactive libraries through a standardized mechanism governed by the [Reactive Streams specification](https://github.com/reactive-streams/reactive-streams-jvm#specification). 2The naming convention of the interface was to append `Source` to the semi-traditional class name. There is no `FlowableSource` since `Publisher` is provided by the Reactive Streams library (and subtyping it wouldn't have helped with interoperation either). These interfaces are, however, not standard in the sense of the Reactive Streams specification and are currently RxJava specific only. +3 **2026.07.09** we haven't decided yet to have a `StreamableSource` becaue we control `Streamable`, unlike `Flow.Publisher`, so we can add defaulted operators there and still have `Streamable` a functional interface. + ### R8 and ProGuard settings By default, RxJava itself doesn't require any ProGuard/R8 settings and should work without problems. Unfortunately, the Reactive Streams dependency since version 1.0.3 has embedded Java 9 class files in its JAR that can cause warnings with the plain ProGuard: