From 948c18a452f2668ea15a3cb5a26bfd580eac45f9 Mon Sep 17 00:00:00 2001 From: maninhill <41712985+maninhill@users.noreply.github.com> Date: Tue, 9 Sep 2025 17:37:12 +0800 Subject: [PATCH 001/932] Add working principle section to README Added a section on the working principle with an image. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 62bfabfd3..f7b21cb76 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,10 @@ SQLBot 是一款基于大模型和 RAG 的智能问数系统。SQLBot 的优势 - **易于集成**: 支持快速嵌入到第三方业务系统，也支持被 n8n、MaxKB、Dify、Coze 等 AI 应用开发平台集成调用，让各类应用快速拥有智能问数能力； - **安全可控**: 提供基于工作空间的资源隔离机制，能够实现细粒度的数据权限控制。 +## 工作原理 + +image + ## 快速开始 ### 安装部署 From 5b324182928d40c6a18db702543202fc76043a5f Mon Sep 17 00:00:00 2001 From: junjun Date: Wed, 10 Sep 2025 10:39:33 +0800 Subject: [PATCH 002/932] refactor: Increase database connection pool --- backend/common/core/db.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/common/core/db.py b/backend/common/core/db.py index 37eff29b7..dd6c97818 100644 --- a/backend/common/core/db.py +++ b/backend/common/core/db.py @@ -1,14 +1,15 @@ from sqlmodel import Session, create_engine, SQLModel - from common.core.config import settings +engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI), pool_size=20, max_overflow=30, pool_recycle=3600, + pool_pre_ping=True) -engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) def get_session(): with Session(engine) as session: yield session + def init_db(): SQLModel.metadata.create_all(engine) From 1976d79796f3cf877290412588d53326f4903bc5 Mon Sep 17 00:00:00 2001 From: junjun Date: Wed, 10 Sep 2025 12:20:36 +0800 Subject: [PATCH 003/932] refactor: Increase database connection pool --- backend/common/core/config.py | 5 +++++ backend/common/core/db.py | 7 +++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/backend/common/core/config.py b/backend/common/core/config.py index de7fc7bb9..043de6ccd 100644 --- a/backend/common/core/config.py +++ b/backend/common/core/config.py @@ -92,5 +92,10 @@ def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn | str: EMBEDDING_SIMILARITY: float = 0.4 EMBEDDING_TOP_COUNT: int = 5 + PG_POOL_SIZE: int = 20 + PG_MAX_OVERFLOW: int = 30 + PG_POOL_RECYCLE: int = 3600 + PG_POOL_PRE_PING: bool = True + settings = Settings() # type: ignore diff --git a/backend/common/core/db.py b/backend/common/core/db.py index dd6c97818..d4adf3461 100644 --- a/backend/common/core/db.py +++ b/backend/common/core/db.py @@ -2,8 +2,11 @@ from common.core.config import settings -engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI), pool_size=20, max_overflow=30, pool_recycle=3600, - pool_pre_ping=True) +engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI), + pool_size=settings.PG_POOL_SIZE, + max_overflow=settings.PG_MAX_OVERFLOW, + pool_recycle=settings.PG_POOL_RECYCLE, + pool_pre_ping=settings.PG_POOL_PRE_PING) def get_session(): From 29ced42e4c6b9f7a0573836f79d438de5159f8c8 Mon Sep 17 00:00:00 2001 From: dataeaseShu Date: Wed, 10 Sep 2025 14:01:08 +0800 Subject: [PATCH 004/932] fix(logo): When you are on other pages, click the top logo to return to the default page of Smart Question, as shown below --- frontend/src/components/layout/LayoutDsl.vue | 36 ++++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/layout/LayoutDsl.vue b/frontend/src/components/layout/LayoutDsl.vue index afedd2db6..f17b81291 100644 --- a/frontend/src/components/layout/LayoutDsl.vue +++ b/frontend/src/components/layout/LayoutDsl.vue @@ -42,6 +42,10 @@ const handleFoldExpand = () => { const toWorkspace = () => { router.push('/') } + +const toChatIndex = () => { + router.push('/chat/index') +} const route = useRoute() const showSysmenu = computed(() => { return route.path.includes('/system') @@ -57,17 +61,37 @@ const showSysmenu = computed(() => { width="30" height="30" :src="logo_fold_blue" - style="margin: 0 0 6px 5px" + @click="toChatIndex" + style="margin: 0 0 6px 5px; cursor: pointer" + /> + - From 1ffd9188965d3d3d82fbe0eca327647ecd541d6a Mon Sep 17 00:00:00 2001 From: xuwei-fit2cloud Date: Wed, 10 Sep 2025 14:35:07 +0800 Subject: [PATCH 005/932] Replace image in README with new source --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f7b21cb76..e66dc40ec 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ SQLBot 是一款基于大模型和 RAG 的智能问数系统。SQLBot 的优势 ## 工作原理 -image +system-arch ## 快速开始 From 35b8a8385ccbf8b2f719d33724ef255f44fbb936 Mon Sep 17 00:00:00 2001 From: xuwei-fit2cloud Date: Wed, 10 Sep 2025 14:45:25 +0800 Subject: [PATCH 006/932] Change image host URL from HTTPS to HTTP --- installer/install.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installer/install.conf b/installer/install.conf index 7aaaabee5..fb525a1da 100644 --- a/installer/install.conf +++ b/installer/install.conf @@ -31,4 +31,4 @@ SQLBOT_LOG_LEVEL="INFO" ## 缓存类型 SQLBOT_CACHE_TYPE="memory" ## MCP 图片存储路径 -SQLBOT_SERVER_IMAGE_HOST=https://YOUR_SERVER_IP:MCP_PORT/images/ +SQLBOT_SERVER_IMAGE_HOST=http://YOUR_SERVER_IP:MCP_PORT/images/ From db8fc429feb34604d374df6a8f6f204c055291a5 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Wed, 10 Sep 2025 18:21:37 +0800 Subject: [PATCH 007/932] fix: The schema field was missed in the Assistant's AES encryption process --- backend/apps/system/crud/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/apps/system/crud/assistant.py b/backend/apps/system/crud/assistant.py index b5d6381b9..01e0c6f00 100644 --- a/backend/apps/system/crud/assistant.py +++ b/backend/apps/system/crud/assistant.py @@ -180,7 +180,7 @@ def convert2schema(self, ds_dict: dict, config: dict[any]) -> AssistantOutDsSche if config.get('encrypt', False): key = config.get('aes_key', None) iv = config.get('aes_iv', None) - aes_attrs = ['host', 'user', 'password', 'dataBase', 'db_schema'] + aes_attrs = ['host', 'user', 'password', 'dataBase', 'db_schema', 'schema'] for attr in aes_attrs: if attr in ds_dict and ds_dict[attr]: try: From 20f2bb927f39f58a5717a244b4aaafe1c4b55489 Mon Sep 17 00:00:00 2001 From: dataeaseShu Date: Thu, 11 Sep 2025 11:38:49 +0800 Subject: [PATCH 008/932] fix: Style optimization --- frontend/src/views/chat/ChatCreator.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/views/chat/ChatCreator.vue b/frontend/src/views/chat/ChatCreator.vue index a70d1f049..93c57f790 100644 --- a/frontend/src/views/chat/ChatCreator.vue +++ b/frontend/src/views/chat/ChatCreator.vue @@ -150,7 +150,7 @@ defineExpose({ - + From ce96e06f77753b72815d0365c1aa61592dd348c2 Mon Sep 17 00:00:00 2001 From: ulleo Date: Thu, 11 Sep 2025 15:57:48 +0800 Subject: [PATCH 009/932] feat: change qwen's default config value --- frontend/src/entity/supplier.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/entity/supplier.ts b/frontend/src/entity/supplier.ts index 75e9434cc..6fee3c595 100644 --- a/frontend/src/entity/supplier.ts +++ b/frontend/src/entity/supplier.ts @@ -43,6 +43,8 @@ export const supplierList: Array<{ { key: 'extra_body', val: '{"enable_thinking": false}', type: 'json' }, ], model_options: [ + { name: 'qwen3-coder-plus' }, + { name: 'qwen3-coder-flash' }, { name: 'qwen-plus' }, /* { name: 'qwen-plus-latest' }, */ { name: 'qwen-max' }, From 6e24ea91f92c4f5f999e7c451011ceb4469ba43a Mon Sep 17 00:00:00 2001 From: junjun Date: Thu, 11 Sep 2025 17:48:26 +0800 Subject: [PATCH 010/932] fix(datasource): fix ClickHouse datasource get tables error --- backend/apps/db/db.py | 2 +- backend/apps/db/db_sql.py | 26 ++++++++++++++++++-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/backend/apps/db/db.py b/backend/apps/db/db.py index 6549740be..fe85de6c4 100644 --- a/backend/apps/db/db.py +++ b/backend/apps/db/db.py @@ -254,7 +254,7 @@ def get_schema(ds: CoreDatasource): def get_tables(ds: CoreDatasource): conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) if ds.type != "excel" else get_engine_config() db = DB.get_db(ds.type) - sql = get_table_sql(ds, conf) + sql = get_table_sql(ds, conf, get_version(ds)) if db.connect_type == ConnectType.sqlalchemy: with get_session(ds) as session: with session.execute(text(sql)) as result: diff --git a/backend/apps/db/db_sql.py b/backend/apps/db/db_sql.py index 34982f6f9..10af38f8b 100644 --- a/backend/apps/db/db_sql.py +++ b/backend/apps/db/db_sql.py @@ -32,7 +32,7 @@ def get_version_sql(ds: CoreDatasource, conf: DatasourceConf): return '' -def get_table_sql(ds: CoreDatasource, conf: DatasourceConf): +def get_table_sql(ds: CoreDatasource, conf: DatasourceConf, db_version: str = ''): if ds.type == "mysql" or ds.type == "doris": return f""" SELECT @@ -95,13 +95,23 @@ def get_table_sql(ds: CoreDatasource, conf: DatasourceConf): ORDER BY t.TABLE_NAME """ elif ds.type == "ck": - return f""" - SELECT name, comment - FROM system.tables - WHERE database = '{conf.database}' - AND engine NOT IN ('Dictionary') - ORDER BY name - """ + version = int(db_version.split('.')[0]) + if version < 22: + return f""" + SELECT name, null as comment + FROM system.tables + WHERE database = '{conf.database}' + AND engine NOT IN ('Dictionary') + ORDER BY name + """ + else: + return f""" + SELECT name, comment + FROM system.tables + WHERE database = '{conf.database}' + AND engine NOT IN ('Dictionary') + ORDER BY name + """ elif ds.type == 'dm': return f""" select table_name, comments From 1e1138263ced90d2a6992cb07cdfa2e7f5d4850b Mon Sep 17 00:00:00 2001 From: junjun Date: Fri, 12 Sep 2025 12:52:22 +0800 Subject: [PATCH 011/932] refactor: Optimize question connection pool --- backend/apps/chat/task/llm.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/backend/apps/chat/task/llm.py b/backend/apps/chat/task/llm.py index ea65ddc8e..584e1dca6 100644 --- a/backend/apps/chat/task/llm.py +++ b/backend/apps/chat/task/llm.py @@ -18,7 +18,7 @@ from langchain_core.messages import BaseMessage, SystemMessage, HumanMessage, AIMessage, BaseMessageChunk from sqlalchemy import select from sqlalchemy.orm import sessionmaker -from sqlmodel import create_engine, Session +from sqlmodel import Session from apps.ai_model.model_factory import LLMConfig, LLMFactory, get_default_config from apps.chat.curd.chat import save_question, save_sql_answer, save_sql, \ @@ -37,6 +37,7 @@ from apps.system.schemas.system_schema import AssistantOutDsSchema from apps.terminology.curd.terminology import get_terminology_template from common.core.config import settings +from common.core.db import engine from common.core.deps import CurrentAssistant, CurrentUser from common.error import SingleMessageError, SQLBotDBError, ParseSQLResultError, SQLBotDBConnectionError from common.utils.utils import SQLBotLogUtil, extract_nested_json, prepare_for_orjson @@ -50,6 +51,10 @@ dynamic_ds_types = [1, 3] dynamic_subsql_prefix = 'select * from sqlbot_dynamic_temp_table_' +session_maker = sessionmaker(bind=engine) +db_session = session_maker() + + class LLMService: ds: CoreDatasource chat_question: ChatQuestion @@ -59,7 +64,7 @@ class LLMService: sql_message: List[Union[BaseMessage, dict[str, Any]]] = [] chart_message: List[Union[BaseMessage, dict[str, Any]]] = [] - session: Session + session: Session = db_session current_user: CurrentUser current_assistant: Optional[CurrentAssistant] = None out_ds_instance: Optional[AssistantOutDs] = None @@ -79,9 +84,9 @@ def __init__(self, current_user: CurrentUser, chat_question: ChatQuestion, current_assistant: Optional[CurrentAssistant] = None, no_reasoning: bool = False, config: LLMConfig = None): self.chunk_list = [] - engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) - session_maker = sessionmaker(bind=engine) - self.session = session_maker() + # engine = create_engine(str(settings.SQLALCHEMY_DATABASE_URI)) + # session_maker = sessionmaker(bind=engine) + # self.session = session_maker() self.session.exec = self.session.exec if hasattr(self.session, "exec") else self.session.execute self.current_user = current_user self.current_assistant = current_assistant From 5ea85fff24598ba20112b177437a83b0a43ea306 Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Fri, 12 Sep 2025 15:07:30 +0800 Subject: [PATCH 012/932] perf: Optimization of the Assistant's History Embedding Logic --- frontend/src/views/embedded/index.vue | 6 +++++- frontend/src/views/embedded/page.vue | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/frontend/src/views/embedded/index.vue b/frontend/src/views/embedded/index.vue index 2f688a230..928ea3f57 100644 --- a/frontend/src/views/embedded/index.vue +++ b/frontend/src/views/embedded/index.vue @@ -137,12 +137,16 @@ onBeforeMount(async () => { const online = route.query.online setFormatOnline(online) + let userFlag = route.query.userFlag + if (userFlag && userFlag === '1') { + userFlag = '100001' + } const now = Date.now() assistantStore.setFlag(now) assistantStore.setId(assistantId?.toString() || '') const param = { id: assistantId, - virtual: assistantStore.getFlag, + virtual: userFlag || assistantStore.getFlag, online, } validator.value = await assistantApi.validate(param) diff --git a/frontend/src/views/embedded/page.vue b/frontend/src/views/embedded/page.vue index decf94757..97aad0035 100644 --- a/frontend/src/views/embedded/page.vue +++ b/frontend/src/views/embedded/page.vue @@ -122,6 +122,10 @@ onBeforeMount(async () => { if (name) { assistantName.value = decodeURIComponent(name.toString()) } + let userFlag = route.query.userFlag + if (userFlag && userFlag === '1') { + userFlag = '100001' + } const now = Date.now() assistantStore.setFlag(now) assistantStore.setId(assistantId?.toString() || '') @@ -132,7 +136,7 @@ onBeforeMount(async () => { } const param = { id: assistantId, - virtual: assistantStore.getFlag, + virtual: userFlag || assistantStore.getFlag, online, } validator.value = await assistantApi.validate(param) From 54b917acd6b5d64ab1926cdcde0d2b79c31c95b3 Mon Sep 17 00:00:00 2001 From: maninhill <41712985+maninhill@users.noreply.github.com> Date: Fri, 12 Sep 2025 18:26:03 +0800 Subject: [PATCH 013/932] Change SQLBot Docker image version to latest Updated Docker image version for SQLBot in README. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e66dc40ec..52083645d 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ docker run -d \ -v ./data/sqlbot/images:/opt/sqlbot/images \ -v ./data/sqlbot/logs:/opt/sqlbot/logs \ -v ./data/postgresql:/var/lib/postgresql/data \ - dataease/sqlbot:v1.1.1 + dataease/sqlbot ``` 你也可以通过 [1Panel 应用商店](https://apps.fit2cloud.com/1panel) 快速部署 SQLBot。 From 10129b785f1743a86c42b21963866eb3eb7d3eb0 Mon Sep 17 00:00:00 2001 From: xuwei-fit2cloud Date: Tue, 16 Sep 2025 09:28:22 +0800 Subject: [PATCH 014/932] Bump version from 1.1.0 to 1.1.3 --- backend/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index b2f693699..6da8e3eb8 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "sqlbot" -version = "1.1.0" +version = "1.1.3" description = "" requires-python = "==3.11.*" dependencies = [ From 0b588c424076b1732b6454d83c25821e5736daaa Mon Sep 17 00:00:00 2001 From: junjun Date: Tue, 16 Sep 2025 10:27:11 +0800 Subject: [PATCH 015/932] feat: support Elasticsearch datasource #108 --- backend/apps/chat/task/llm.py | 24 +++--- backend/apps/db/constant.py | 1 + backend/apps/db/db.py | 37 ++++++++- backend/apps/db/es_engine.py | 84 +++++++++++++++++++++ backend/apps/db/type.py | 3 +- backend/pyproject.toml | 1 + backend/template.yaml | 4 +- frontend/src/assets/datasource/icon_es.png | Bin 0 -> 1035 bytes frontend/src/i18n/en.json | 3 +- frontend/src/i18n/zh-CN.json | 3 +- frontend/src/views/ds/DatasourceForm.vue | 24 ++++-- frontend/src/views/ds/js/ds-type.ts | 3 + 12 files changed, 163 insertions(+), 24 deletions(-) create mode 100644 backend/apps/db/es_engine.py create mode 100644 frontend/src/assets/datasource/icon_es.png diff --git a/backend/apps/chat/task/llm.py b/backend/apps/chat/task/llm.py index 584e1dca6..aa4eff581 100644 --- a/backend/apps/chat/task/llm.py +++ b/backend/apps/chat/task/llm.py @@ -625,7 +625,7 @@ def generate_assistant_dynamic_sql(self, sql, tables: List): result_dict = {} for table in ds.tables: if table.name in tables and table.sql: - #sub_query.append({"table": table.name, "query": table.sql}) + # sub_query.append({"table": table.name, "query": table.sql}) result_dict[table.name] = table.sql sub_query.append({"table": table.name, "query": f'{dynamic_subsql_prefix}{table.name}'}) if not sub_query: @@ -881,7 +881,7 @@ def save_sql_data(self, data_obj: Dict[str, Any]): def finish(self): return finish_record(session=self.session, record_id=self.record.id) - def execute_sql(self, sql: str): + def execute_sql(self, sql: str, tables): """Execute SQL query Args: @@ -893,7 +893,7 @@ def execute_sql(self, sql: str): """ SQLBotLogUtil.info(f"Executing SQL on ds_id {self.ds.id}: {sql}") try: - return exec_sql(self.ds, sql) + return exec_sql(ds=self.ds, sql=sql, origin_column=False, table_name=tables) except Exception as e: if isinstance(e, ParseSQLResultError): raise e @@ -1000,14 +1000,16 @@ def run_task(self, in_chat: bool = True): sqlbot_temp_sql_text = None assistant_dynamic_sql = None # todo row permission - if ((not self.current_assistant or is_page_embedded) and is_normal_user(self.current_user)) or use_dynamic_ds: + if ((not self.current_assistant or is_page_embedded) and is_normal_user( + self.current_user)) or use_dynamic_ds: sql, tables = self.check_sql(res=full_sql_text) sql_result = None - + if use_dynamic_ds: dynamic_sql_result = self.generate_assistant_dynamic_sql(sql, tables) - sqlbot_temp_sql_text = dynamic_sql_result.get('sqlbot_temp_sql_text') if dynamic_sql_result else None - #sql_result = self.generate_assistant_filter(sql, tables) + sqlbot_temp_sql_text = dynamic_sql_result.get( + 'sqlbot_temp_sql_text') if dynamic_sql_result else None + # sql_result = self.generate_assistant_filter(sql, tables) else: sql_result = self.generate_filter(sql, tables) # maybe no sql and tables @@ -1020,6 +1022,7 @@ def run_task(self, in_chat: bool = True): sql = self.check_save_sql(res=full_sql_text) else: sql = self.check_save_sql(res=full_sql_text) + tables = [] SQLBotLogUtil.info(sql) format_sql = sqlparse.format(sql, reindent=True) @@ -1033,10 +1036,11 @@ def run_task(self, in_chat: bool = True): if sqlbot_temp_sql_text and assistant_dynamic_sql: dynamic_sql_result.pop('sqlbot_temp_sql_text') for origin_table, subsql in dynamic_sql_result.items(): - assistant_dynamic_sql = assistant_dynamic_sql.replace(f'{dynamic_subsql_prefix}{origin_table}', subsql) + assistant_dynamic_sql = assistant_dynamic_sql.replace(f'{dynamic_subsql_prefix}{origin_table}', + subsql) real_execute_sql = assistant_dynamic_sql - - result = self.execute_sql(sql=real_execute_sql) + + result = self.execute_sql(sql=real_execute_sql, tables=tables) self.save_sql_data(data_obj=result) if in_chat: yield 'data:' + orjson.dumps({'content': 'execute-success', 'type': 'sql-data'}).decode() + '\n\n' diff --git a/backend/apps/db/constant.py b/backend/apps/db/constant.py index c4de1cd0f..67074ac12 100644 --- a/backend/apps/db/constant.py +++ b/backend/apps/db/constant.py @@ -22,6 +22,7 @@ class DB(Enum): dm = ('dm', '"', '"', ConnectType.py_driver) doris = ('doris', '`', '`', ConnectType.py_driver) redshift = ('redshift', '"', '"', ConnectType.py_driver) + es = ('es', '"', '"', ConnectType.py_driver) def __init__(self, type, prefix, suffix, connect_type: ConnectType): self.type = type diff --git a/backend/apps/db/db.py b/backend/apps/db/db.py index fe85de6c4..41cf503ac 100644 --- a/backend/apps/db/db.py +++ b/backend/apps/db/db.py @@ -24,6 +24,7 @@ from common.core.deps import Trans from common.utils.utils import SQLBotLogUtil from fastapi import HTTPException +from apps.db.es_engine import get_es_connect, get_es_index, get_es_fields, get_es_data def get_uri(ds: CoreDatasource) -> str: @@ -144,7 +145,8 @@ def check_connection(trans: Optional[Trans], ds: CoreDatasource | AssistantOutDs raise HTTPException(status_code=500, detail=trans('i18n_ds_invalid') + f': {e.args}') return False elif ds.type == 'redshift': - with redshift_connector.connect(host=conf.host, port=conf.port, database=conf.database, user=conf.username, + with redshift_connector.connect(host=conf.host, port=conf.port, database=conf.database, + user=conf.username, password=conf.password, timeout=10) as conn, conn.cursor() as cursor: try: @@ -156,6 +158,14 @@ def check_connection(trans: Optional[Trans], ds: CoreDatasource | AssistantOutDs if is_raise: raise HTTPException(status_code=500, detail=trans('i18n_ds_invalid') + f': {e.args}') return False + elif ds.type == 'es': + es_conn = get_es_connect(conf) + if es_conn.ping(): + SQLBotLogUtil.info("success") + return True + else: + SQLBotLogUtil.info("failed") + return False else: conn = get_ds_engine(ds) try: @@ -208,7 +218,7 @@ def get_version(ds: CoreDatasource | AssistantOutDsSchema): cursor.execute(sql) res = cursor.fetchall() version = res[0][0] - elif ds.type == 'redshift': + elif ds.type == 'redshift' or ds.type == 'es': version = '' except Exception as e: print(e) @@ -285,6 +295,10 @@ def get_tables(ds: CoreDatasource): res = cursor.fetchall() res_list = [TableSchema(*item) for item in res] return res_list + elif ds.type == 'es': + res = get_es_index(conf) + res_list = [TableSchema(*item) for item in res] + return res_list def get_fields(ds: CoreDatasource, table_name: str = None): @@ -321,9 +335,13 @@ def get_fields(ds: CoreDatasource, table_name: str = None): res = cursor.fetchall() res_list = [ColumnSchema(*item) for item in res] return res_list + elif ds.type == 'es': + res = get_es_fields(conf, table_name) + res_list = [ColumnSchema(*item) for item in res] + return res_list -def exec_sql(ds: CoreDatasource | AssistantOutDsSchema, sql: str, origin_column=False): +def exec_sql(ds: CoreDatasource | AssistantOutDsSchema, sql: str, origin_column=False, table_name=None): while sql.endswith(';'): sql = sql[:-1] @@ -401,3 +419,16 @@ def exec_sql(ds: CoreDatasource | AssistantOutDsSchema, sql: str, origin_column= "sql": bytes.decode(base64.b64encode(bytes(sql, 'utf-8')))} except Exception as ex: raise ParseSQLResultError(str(ex)) + elif ds.type == 'es': + if table_name and table_name[0]: + res, columns = get_es_data(conf, sql, table_name[0]) + columns = [field[0] for field in columns] if origin_column else [field[0].lower() for + field in + columns] + result_list = [ + {str(columns[i]): float(value) if isinstance(value, Decimal) else value for i, value in + enumerate(tuple_item)} + for tuple_item in res + ] + return {"fields": columns, "data": result_list, + "sql": bytes.decode(base64.b64encode(bytes(sql, 'utf-8')))} diff --git a/backend/apps/db/es_engine.py b/backend/apps/db/es_engine.py new file mode 100644 index 000000000..897251ed5 --- /dev/null +++ b/backend/apps/db/es_engine.py @@ -0,0 +1,84 @@ +# Author: Junjun +# Date: 2025/9/9 + +import json + +import requests +from elasticsearch import Elasticsearch + +from apps.datasource.models.datasource import DatasourceConf + + +def get_es_connect(conf: DatasourceConf): + es_client = Elasticsearch( + [conf.host], # ES address + basic_auth=(conf.username, conf.password), + verify_certs=False, + compatibility_mode=True + ) + return es_client + + +# get tables +def get_es_index(conf: DatasourceConf): + es_client = get_es_connect(conf) + indices = es_client.cat.indices(format="json") + res = [] + for idx in indices: + index_name = idx.get('index') + desc = '' + # get mapping + mapping = es_client.indices.get_mapping(index=index_name) + mappings = mapping.get(index_name).get("mappings") + if mappings.get('_meta'): + desc = mappings.get('_meta').get('description') + res.append((index_name, desc)) + return res + + +# get fields +def get_es_fields(conf: DatasourceConf, table_name: str): + es_client = get_es_connect(conf) + index_name = table_name + mapping = es_client.indices.get_mapping(index=index_name) + properties = mapping.get(index_name).get("mappings").get("properties") + res = [] + for field, config in properties.items(): + field_type = config.get("type") + desc = '' + if config.get("_meta"): + desc = config.get("_meta").get('description') + + if field_type: + res.append((field, field_type, desc)) + else: + # object、nested... + res.append((field, ','.join(list(config.keys())), desc)) + return res + + +def get_es_data(conf: DatasourceConf, sql: str, table_name: str): + r = requests.post(f"{conf.host}/_sql/translate", json={"query": sql}) + # print(json.dumps(r.json())) + + es_client = get_es_connect(conf) + response = es_client.search( + index=table_name, + body=json.dumps(r.json()) + ) + + # print(response) + fields = get_es_fields(conf, table_name) + res = [] + for hit in response.get('hits').get('hits'): + item = [] + if 'fields' in hit: + result = hit.get('fields') # {'title': ['Python'], 'age': [30]} + for field in fields: + v = result.get(field[0]) + item.append(v[0]) if v else item.append(None) + res.append(tuple(item)) + # print(hit['fields']['title'][0]) + # elif '_source' in hit: + # print(hit.get('_source')) + return res, fields diff --git a/backend/apps/db/type.py b/backend/apps/db/type.py index e2c6c4fb7..1e48dc654 100644 --- a/backend/apps/db/type.py +++ b/backend/apps/db/type.py @@ -13,5 +13,6 @@ def db_type_relation() -> Dict: "ck": "ClickHouse", "dm": "达梦", "doris": "Apache Doris", - "redshift": "AWS Redshift" + "redshift": "AWS Redshift", + "es": "Elasticsearch" } diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 6da8e3eb8..a1c5b4e32 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -50,6 +50,7 @@ dependencies = [ "dicttoxml>=1.7.16", "dmpython>=2.5.22; platform_system != 'Darwin'", "redshift-connector>=2.1.8", + "elasticsearch[requests] (>=7.10,<8.0)", ] [project.optional-dependencies] diff --git a/backend/template.yaml b/backend/template.yaml index dd182300d..49f07c1ef 100644 --- a/backend/template.yaml +++ b/backend/template.yaml @@ -70,7 +70,7 @@ template: 生成SQL时，必须避免与数据库关键字冲突 - 如数据库引擎是 PostgreSQL、Oracle、ClickHouse、达梦（DM）、AWS Redshift，则在schema、表名、字段名、别名外层加双引号； + 如数据库引擎是 PostgreSQL、Oracle、ClickHouse、达梦（DM）、AWS Redshift、Elasticsearch，则在schema、表名、字段名、别名外层加双引号； 如数据库引擎是 MySQL、Doris，则在表名、字段名、别名外层加反引号； 如数据库引擎是 Microsoft SQL Server，则在schema、表名、字段名、别名外层加方括号。 @@ -448,7 +448,7 @@ template: - 如果存在冗余的过滤条件则进行去重后再生成新SQL。 - 给过滤条件中的字段前加上表别名（如果没有表别名则加表名），如：table.field。 - 生成SQL时，必须避免关键字冲突: - - 如数据库引擎是 PostgreSQL、Oracle、ClickHouse、达梦（DM）、AWS Redshift，则在schema、表名、字段名、别名外层加双引号； + - 如数据库引擎是 PostgreSQL、Oracle、ClickHouse、达梦（DM）、AWS Redshift、Elasticsearch，则在schema、表名、字段名、别名外层加双引号； - 如数据库引擎是 MySQL、Doris，则在表名、字段名、别名外层加反引号； - 如数据库引擎是 Microsoft SQL Server，则在schema、表名、字段名、别名外层加方括号。 - 生成的SQL使用JSON格式返回： diff --git a/frontend/src/assets/datasource/icon_es.png b/frontend/src/assets/datasource/icon_es.png new file mode 100644 index 0000000000000000000000000000000000000000..5d8f6e6e4e8420b20291f4a42a733b25d2440126 GIT binary patch literal 1035 zcmV+m1oZofP)I}aI#Q@eC>pQGC7l%3C9B*&dYQ4K{z{{ z!%^0^HvZeQUAwRA`u*?wexB#PVNr_$P}Kxgtyk3!^$)u>Fyc|wQ>yxws>c5ppj}lH zXX5)c=y95^J!)PkRI@wr_H${K>d=>}rtiF}7RF*p1UMh%+hE^Ft@;93mS_J0+oh&!f%s#*w`e@=XQtipeG0DvW%Du}OXg^{oByyVP!Bxmo( zBGM3v%f5;Bcp`gZn79G#Q3k4-}$JG2ke|JF^0XG<1?dstFq$QwKB+ieO6AU{eTpmfRv{Qec7om(q(I=s&Rme zL*7u|zP2HN?V&C5nc7|P)6@fFJ`EK;eQzA^l=;}z-HyvU1;EOr6t?FTvN6>|y!SLm zho581_q})`W@iX^KUv*ti!+J<46CsJ-4bi4;N>CFpz^eq6WC)9E_Mz<39MD{3-fv2$%|`l!_~B5R_wDtRfx6aR#b2pRU}CU7mLfrtb% zZA-r1kr(ss2&pnPouK*wh%;pOy6~>^BII0Z2&d{D}ZU>0?;fX3%G$6wW$A<{soDKliIJjFF*hQ002ovPDHLk FV1kOo`s4rr literal 0 HcmV?d00001 diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json index c0eef984c..ea6d621c1 100644 --- a/frontend/src/i18n/en.json +++ b/frontend/src/i18n/en.json @@ -256,7 +256,8 @@ "success": "Connect success", "failed": "Connect failed" }, - "timeout": "Timeout(second)" + "timeout": "Timeout(second)", + "address": "Address" } }, "datasource": { diff --git a/frontend/src/i18n/zh-CN.json b/frontend/src/i18n/zh-CN.json index 4cadc4c2c..192f68274 100644 --- a/frontend/src/i18n/zh-CN.json +++ b/frontend/src/i18n/zh-CN.json @@ -256,7 +256,8 @@ "success": "连接成功", "failed": "连接失败" }, - "timeout": "查询超时(秒)" + "timeout": "查询超时(秒)", + "address": "地址" } }, "datasource": { diff --git a/frontend/src/views/ds/DatasourceForm.vue b/frontend/src/views/ds/DatasourceForm.vue index 47d9919ac..72e82340a 100644 --- a/frontend/src/views/ds/DatasourceForm.vue +++ b/frontend/src/views/ds/DatasourceForm.vue @@ -513,6 +513,7 @@ defineExpose({ 12+ 5.6+ 9.6+ + 7+
@@ -594,14 +595,21 @@ defineExpose({ />
- + - + - + {{ t('ds.form.mode.sid') }} - + - + Date: Tue, 16 Sep 2025 10:49:33 +0800 Subject: [PATCH 016/932] feat: support Elasticsearch datasource #108 --- backend/apps/db/es_engine.py | 40 +++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/backend/apps/db/es_engine.py b/backend/apps/db/es_engine.py index 897251ed5..a82f15897 100644 --- a/backend/apps/db/es_engine.py +++ b/backend/apps/db/es_engine.py @@ -24,15 +24,16 @@ def get_es_index(conf: DatasourceConf): es_client = get_es_connect(conf) indices = es_client.cat.indices(format="json") res = [] - for idx in indices: - index_name = idx.get('index') - desc = '' - # get mapping - mapping = es_client.indices.get_mapping(index=index_name) - mappings = mapping.get(index_name).get("mappings") - if mappings.get('_meta'): - desc = mappings.get('_meta').get('description') - res.append((index_name, desc)) + if indices is not None: + for idx in indices: + index_name = idx.get('index') + desc = '' + # get mapping + mapping = es_client.indices.get_mapping(index=index_name) + mappings = mapping.get(index_name).get("mappings") + if mappings.get('_meta'): + desc = mappings.get('_meta').get('description') + res.append((index_name, desc)) return res @@ -43,17 +44,18 @@ def get_es_fields(conf: DatasourceConf, table_name: str): mapping = es_client.indices.get_mapping(index=index_name) properties = mapping.get(index_name).get("mappings").get("properties") res = [] - for field, config in properties.items(): - field_type = config.get("type") - desc = '' - if config.get("_meta"): - desc = config.get("_meta").get('description') + if properties is not None: + for field, config in properties.items(): + field_type = config.get("type") + desc = '' + if config.get("_meta"): + desc = config.get("_meta").get('description') - if field_type: - res.append((field, field_type, desc)) - else: - # object、nested... - res.append((field, ','.join(list(config.keys())), desc)) + if field_type: + res.append((field, field_type, desc)) + else: + # object、nested... + res.append((field, ','.join(list(config.keys())), desc)) return res From 44362635f6897f9fdedbcc095622dfaa6f3b85d1 Mon Sep 17 00:00:00 2001 From: junjun Date: Tue, 16 Sep 2025 11:32:25 +0800 Subject: [PATCH 017/932] fix: mcp images path error --- backend/apps/chat/task/llm.py | 2 +- backend/common/core/config.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/apps/chat/task/llm.py b/backend/apps/chat/task/llm.py index aa4eff581..16736aa97 100644 --- a/backend/apps/chat/task/llm.py +++ b/backend/apps/chat/task/llm.py @@ -1274,7 +1274,7 @@ def request_picture(chat_id: int, record_id: int, chart: dict, data: dict): requests.post(url=settings.MCP_IMAGE_HOST, json=request_obj) - request_path = urllib.parse.urljoin(settings.MCP_IMAGE_HOST, f"{file_name}.png") + request_path = urllib.parse.urljoin(settings.SERVER_IMAGE_HOST, f"{file_name}.png") return request_path diff --git a/backend/common/core/config.py b/backend/common/core/config.py index 043de6ccd..6a922ae96 100644 --- a/backend/common/core/config.py +++ b/backend/common/core/config.py @@ -84,7 +84,7 @@ def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn | str: MCP_IMAGE_PATH: str = '/opt/sqlbot/images' EXCEL_PATH: str = '/opt/sqlbot/data/excel' MCP_IMAGE_HOST: str = 'http://localhost:3000' - SERVER_IMAGE_HOST: str = 'https://YOUR_SERVE_IP:MCP_PORT/images/' + SERVER_IMAGE_HOST: str = 'http://YOUR_SERVE_IP:MCP_PORT/images/' LOCAL_MODEL_PATH: str = '/opt/sqlbot/models' DEFAULT_EMBEDDING_MODEL: str = 'shibing624/text2vec-base-chinese' From 2a1205dcbf01f95a3aada8bf2ff060530ebb3d35 Mon Sep 17 00:00:00 2001 From: junjun Date: Tue, 16 Sep 2025 11:41:12 +0800 Subject: [PATCH 018/932] feat: support Elasticsearch datasource #108 --- backend/apps/datasource/crud/datasource.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index eb62f54a7..2de6a587f 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -293,6 +293,10 @@ def preview(session: SessionDep, current_user: CurrentUser, id: int, data: Table sql = f"""SELECT "{'", "'.join(fields)}" FROM "{conf.dbSchema}"."{data.table.table_name}" {where} LIMIT 100""" + elif ds.type == "es": + sql = f"""SELECT "{'", "'.join(fields)}" FROM "{data.table.table_name}" + {where} + LIMIT 100""" return exec_sql(ds, sql, True) From 0cb426de7178890acc9f23b8df7162f86802822b Mon Sep 17 00:00:00 2001 From: junjun Date: Tue, 16 Sep 2025 13:44:48 +0800 Subject: [PATCH 019/932] feat: support Elasticsearch datasource #108 --- backend/apps/datasource/api/datasource.py | 1 + backend/apps/datasource/crud/datasource.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/apps/datasource/api/datasource.py b/backend/apps/datasource/api/datasource.py index 60dd929e8..69cad28f3 100644 --- a/backend/apps/datasource/api/datasource.py +++ b/backend/apps/datasource/api/datasource.py @@ -135,6 +135,7 @@ class TestObj(BaseModel): sql: str = None +# not used, just do test @router.post("/execSql/{id}") async def exec_sql(session: SessionDep, id: int, obj: TestObj): def inner(): diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index 2de6a587f..b4720bad3 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -297,7 +297,7 @@ def preview(session: SessionDep, current_user: CurrentUser, id: int, data: Table sql = f"""SELECT "{'", "'.join(fields)}" FROM "{data.table.table_name}" {where} LIMIT 100""" - return exec_sql(ds, sql, True) + return exec_sql(ds, sql, True, [data.table.table_name]) def fieldEnum(session: SessionDep, id: int): @@ -313,7 +313,7 @@ def fieldEnum(session: SessionDep, id: int): db = DB.get_db(ds.type) sql = f"""SELECT DISTINCT {db.prefix}{field.field_name}{db.suffix} FROM {db.prefix}{table.table_name}{db.suffix}""" - res = exec_sql(ds, sql, True) + res = exec_sql(ds, sql, True, [table.table_name]) return [item.get(res.get('fields')[0]) for item in res.get('data')] @@ -353,7 +353,7 @@ def get_table_schema(session: SessionDep, current_user: CurrentUser, ds: CoreDat db_name = table_objs[0].schema schema_str += f"【DB_ID】 {db_name}\n【Schema】\n" for obj in table_objs: - schema_str += f"# Table: {db_name}.{obj.table.table_name}" if ds.type != "mysql" else f"# Table: {obj.table.table_name}" + schema_str += f"# Table: {db_name}.{obj.table.table_name}" if ds.type != "mysql" and ds.type != "es" else f"# Table: {obj.table.table_name}" table_comment = '' if obj.table.custom_comment: table_comment = obj.table.custom_comment.strip() From ae2b9a2aedf18fe4f05e276fef7e29ae0d1e462f Mon Sep 17 00:00:00 2001 From: xuwei-fit2cloud Date: Tue, 16 Sep 2025 13:54:33 +0800 Subject: [PATCH 020/932] Add --privileged flag to SQLBot deployment command --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 52083645d..98e00c880 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ docker run -d \ -v ./data/sqlbot/images:/opt/sqlbot/images \ -v ./data/sqlbot/logs:/opt/sqlbot/logs \ -v ./data/postgresql:/var/lib/postgresql/data \ + --privileged=true \ dataease/sqlbot ``` From bf43221e6aa77803081005920feaefa1d967d4fc Mon Sep 17 00:00:00 2001 From: junjun Date: Tue, 16 Sep 2025 14:24:29 +0800 Subject: [PATCH 021/932] feat: support Elasticsearch datasource #108 --- backend/apps/db/es_engine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/apps/db/es_engine.py b/backend/apps/db/es_engine.py index a82f15897..2373af999 100644 --- a/backend/apps/db/es_engine.py +++ b/backend/apps/db/es_engine.py @@ -5,6 +5,7 @@ import requests from elasticsearch import Elasticsearch +from fastapi import HTTPException from apps.datasource.models.datasource import DatasourceConf @@ -61,7 +62,8 @@ def get_es_fields(conf: DatasourceConf, table_name: str): def get_es_data(conf: DatasourceConf, sql: str, table_name: str): r = requests.post(f"{conf.host}/_sql/translate", json={"query": sql}) - # print(json.dumps(r.json())) + if r.json().get('error'): + raise HTTPException(status_code=500, detail=f': {json.dumps(r.json())}') es_client = get_es_connect(conf) response = es_client.search( From 5cb48b9f602dc48ad9cd587c91acb75e931874cd Mon Sep 17 00:00:00 2001 From: dataeaseShu Date: Tue, 16 Sep 2025 14:38:10 +0800 Subject: [PATCH 022/932] feat(Embedded Management): Exposing the assistant ID for three-party docking --- frontend/src/views/system/embedded/Card.vue | 51 ++++++++++++++++++--- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/frontend/src/views/system/embedded/Card.vue b/frontend/src/views/system/embedded/Card.vue index 4519d25d3..4b496d92a 100644 --- a/frontend/src/views/system/embedded/Card.vue +++ b/frontend/src/views/system/embedded/Card.vue @@ -1,5 +1,6 @@ + + + + + From 3ecf2a0c182a0aa45b6d2b30a6a8f1559cea7eb3 Mon Sep 17 00:00:00 2001 From: ulleo Date: Wed, 17 Sep 2025 14:36:32 +0800 Subject: [PATCH 038/932] feat: add SQL examples in prompt --- backend/apps/chat/task/llm.py | 4 ++++ backend/apps/data_training/curd/data_training.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/apps/chat/task/llm.py b/backend/apps/chat/task/llm.py index ae2eb9f98..5455187da 100644 --- a/backend/apps/chat/task/llm.py +++ b/backend/apps/chat/task/llm.py @@ -29,6 +29,7 @@ get_chat_chart_data, list_generate_sql_logs, list_generate_chart_logs, start_log, end_log, \ get_last_execute_sql_error from apps.chat.models.chat_model import ChatQuestion, ChatRecord, Chat, RenameChat, ChatLog, OperationEnum +from apps.data_training.curd.data_training import get_training_template from apps.datasource.crud.datasource import get_table_schema from apps.datasource.crud.permission import get_row_permission_filters, is_normal_user from apps.datasource.models.datasource import CoreDatasource @@ -935,6 +936,9 @@ def run_task(self, in_chat: bool = True): self.chat_question.terminologies = get_terminology_template(self.session, self.chat_question.question, self.ds.oid if isinstance(self.ds, CoreDatasource) else 1) + self.chat_question.data_training = get_training_template(self.session, self.chat_question.question, + self.ds.id, self.ds.oid) + self.init_messages() # return id diff --git a/backend/apps/data_training/curd/data_training.py b/backend/apps/data_training/curd/data_training.py index 2ef63361f..c99b02eeb 100644 --- a/backend/apps/data_training/curd/data_training.py +++ b/backend/apps/data_training/curd/data_training.py @@ -204,7 +204,7 @@ def save_embeddings(session: Session, ids: List[int]): """ -def select_terminology_by_question(session: SessionDep, question: str, oid: int, datasource: int): +def select_training_by_question(session: SessionDep, question: str, oid: int, datasource: int): if question.strip() == "": return [] @@ -303,7 +303,7 @@ def get_training_template(session: SessionDep, question: str, datasource: int, o oid = 1 if not datasource: return '' - _results = select_terminology_by_question(session, question, oid, datasource) + _results = select_training_by_question(session, question, oid, datasource) if _results and len(_results) > 0: data_training = to_xml_string(_results) template = get_base_data_training_template().format(data_training=data_training) From 24f23b460ca686bb27003d269dc5375b9901b811 Mon Sep 17 00:00:00 2001 From: junjun Date: Wed, 17 Sep 2025 14:45:49 +0800 Subject: [PATCH 039/932] fix: Fix possible SQL injection vulnerabilities --- backend/apps/db/db.py | 20 ++++---- backend/apps/db/db_sql.py | 100 +++++++++++++++++++------------------- 2 files changed, 60 insertions(+), 60 deletions(-) diff --git a/backend/apps/db/db.py b/backend/apps/db/db.py index 5018c9947..9c1b2cfaf 100644 --- a/backend/apps/db/db.py +++ b/backend/apps/db/db.py @@ -264,10 +264,10 @@ def get_schema(ds: CoreDatasource): def get_tables(ds: CoreDatasource): conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) if ds.type != "excel" else get_engine_config() db = DB.get_db(ds.type) - sql = get_table_sql(ds, conf, get_version(ds)) + sql, sql_param = get_table_sql(ds, conf, get_version(ds)) if db.connect_type == ConnectType.sqlalchemy: with get_session(ds) as session: - with session.execute(text(sql)) as result: + with session.execute(text(sql), {"param": sql_param}) as result: res = result.fetchall() res_list = [TableSchema(*item) for item in res] return res_list @@ -275,7 +275,7 @@ def get_tables(ds: CoreDatasource): if ds.type == 'dm': with dmPython.connect(user=conf.username, password=conf.password, server=conf.host, port=conf.port) as conn, conn.cursor() as cursor: - cursor.execute(sql, timeout=conf.timeout) + cursor.execute(sql, {"param": sql_param}, timeout=conf.timeout) res = cursor.fetchall() res_list = [TableSchema(*item) for item in res] return res_list @@ -283,7 +283,7 @@ def get_tables(ds: CoreDatasource): with pymysql.connect(user=conf.username, passwd=conf.password, host=conf.host, port=conf.port, db=conf.database, connect_timeout=conf.timeout, read_timeout=conf.timeout) as conn, conn.cursor() as cursor: - cursor.execute(sql) + cursor.execute(sql, {"param": sql_param}) res = cursor.fetchall() res_list = [TableSchema(*item) for item in res] return res_list @@ -291,7 +291,7 @@ def get_tables(ds: CoreDatasource): with redshift_connector.connect(host=conf.host, port=conf.port, database=conf.database, user=conf.username, password=conf.password, timeout=conf.timeout) as conn, conn.cursor() as cursor: - cursor.execute(sql) + cursor.execute(sql, {"param": sql_param}) res = cursor.fetchall() res_list = [TableSchema(*item) for item in res] return res_list @@ -304,10 +304,10 @@ def get_tables(ds: CoreDatasource): def get_fields(ds: CoreDatasource, table_name: str = None): conf = DatasourceConf(**json.loads(aes_decrypt(ds.configuration))) if ds.type != "excel" else get_engine_config() db = DB.get_db(ds.type) - sql = get_field_sql(ds, conf, table_name) + sql, p1, p2 = get_field_sql(ds, conf, table_name) if db.connect_type == ConnectType.sqlalchemy: with get_session(ds) as session: - with session.execute(text(sql)) as result: + with session.execute(text(sql), {"param1": p1, "param2": p2}) as result: res = result.fetchall() res_list = [ColumnSchema(*item) for item in res] return res_list @@ -315,7 +315,7 @@ def get_fields(ds: CoreDatasource, table_name: str = None): if ds.type == 'dm': with dmPython.connect(user=conf.username, password=conf.password, server=conf.host, port=conf.port) as conn, conn.cursor() as cursor: - cursor.execute(sql, timeout=conf.timeout) + cursor.execute(sql, {"param1": p1, "param2": p2}, timeout=conf.timeout) res = cursor.fetchall() res_list = [ColumnSchema(*item) for item in res] return res_list @@ -323,7 +323,7 @@ def get_fields(ds: CoreDatasource, table_name: str = None): with pymysql.connect(user=conf.username, passwd=conf.password, host=conf.host, port=conf.port, db=conf.database, connect_timeout=conf.timeout, read_timeout=conf.timeout) as conn, conn.cursor() as cursor: - cursor.execute(sql) + cursor.execute(sql, {"param1": p1, "param2": p2}) res = cursor.fetchall() res_list = [ColumnSchema(*item) for item in res] return res_list @@ -331,7 +331,7 @@ def get_fields(ds: CoreDatasource, table_name: str = None): with redshift_connector.connect(host=conf.host, port=conf.port, database=conf.database, user=conf.username, password=conf.password, timeout=conf.timeout) as conn, conn.cursor() as cursor: - cursor.execute(sql) + cursor.execute(sql, {"param1": p1, "param2": p2}) res = cursor.fetchall() res_list = [ColumnSchema(*item) for item in res] return res_list diff --git a/backend/apps/db/db_sql.py b/backend/apps/db/db_sql.py index 10af38f8b..f54837791 100644 --- a/backend/apps/db/db_sql.py +++ b/backend/apps/db/db_sql.py @@ -34,17 +34,17 @@ def get_version_sql(ds: CoreDatasource, conf: DatasourceConf): def get_table_sql(ds: CoreDatasource, conf: DatasourceConf, db_version: str = ''): if ds.type == "mysql" or ds.type == "doris": - return f""" + return """ SELECT TABLE_NAME, TABLE_COMMENT FROM information_schema.TABLES WHERE - TABLE_SCHEMA = '{conf.database}' - """ + TABLE_SCHEMA = :param + """, conf.database elif ds.type == "sqlServer": - return f""" + return """ SELECT TABLE_NAME AS [TABLE_NAME], ISNULL(ep.value, '') AS [TABLE_COMMENT] @@ -57,10 +57,10 @@ def get_table_sql(ds: CoreDatasource, conf: DatasourceConf, db_version: str = '' AND ep.name = 'MS_Description' WHERE t.TABLE_TYPE IN ('BASE TABLE', 'VIEW') - AND t.TABLE_SCHEMA = '{conf.dbSchema}' - """ + AND t.TABLE_SCHEMA = :param + """, conf.dbSchema elif ds.type == "pg" or ds.type == "excel": - return f""" + return """ SELECT c.relname AS TABLE_NAME, COALESCE(d.description, obj_description(c.oid)) AS TABLE_COMMENT FROM pg_class c @@ -68,59 +68,59 @@ def get_table_sql(ds: CoreDatasource, conf: DatasourceConf, db_version: str = '' pg_namespace n ON n.oid = c.relnamespace LEFT JOIN pg_description d ON d.objoid = c.oid AND d.objsubid = 0 - WHERE n.nspname = '{conf.dbSchema}' + WHERE n.nspname = :param AND c.relkind IN ('r', 'v', 'p', 'm') AND c.relname NOT LIKE 'pg_%' AND c.relname NOT LIKE 'sql_%' ORDER BY c.relname \ - """ + """, conf.dbSchema elif ds.type == "oracle": - return f""" + return """ SELECT t.TABLE_NAME AS "TABLE_NAME", NVL(c.COMMENTS, '') AS "TABLE_COMMENT" FROM ( SELECT TABLE_NAME, 'TABLE' AS OBJECT_TYPE FROM DBA_TABLES - WHERE OWNER = '{conf.dbSchema}' + WHERE OWNER = :param UNION ALL SELECT VIEW_NAME AS TABLE_NAME, 'VIEW' AS OBJECT_TYPE FROM DBA_VIEWS - WHERE OWNER = '{conf.dbSchema}' + WHERE OWNER = :param ) t LEFT JOIN DBA_TAB_COMMENTS c ON t.TABLE_NAME = c.TABLE_NAME AND c.TABLE_TYPE = t.OBJECT_TYPE - AND c.OWNER = '{conf.dbSchema}' + AND c.OWNER = :param ORDER BY t.TABLE_NAME - """ + """, conf.dbSchema elif ds.type == "ck": version = int(db_version.split('.')[0]) if version < 22: - return f""" + return """ SELECT name, null as comment FROM system.tables - WHERE database = '{conf.database}' + WHERE database = :param AND engine NOT IN ('Dictionary') ORDER BY name - """ + """, conf.database else: - return f""" + return """ SELECT name, comment FROM system.tables - WHERE database = '{conf.database}' + WHERE database = :param AND engine NOT IN ('Dictionary') ORDER BY name - """ + """, conf.database elif ds.type == 'dm': - return f""" + return """ select table_name, comments from all_tab_comments - where owner='{conf.dbSchema}' + where owner=:param AND (table_type = 'TABLE' or table_type = 'VIEW') - """ + """, conf.dbSchema elif ds.type == 'redshift': - return f""" + return """ SELECT relname AS TableName, obj_description(relfilenode::regclass, 'pg_class') AS TableDescription @@ -128,13 +128,13 @@ def get_table_sql(ds: CoreDatasource, conf: DatasourceConf, db_version: str = '' pg_class WHERE relkind in ('r','p', 'f') - AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = '{conf.dbSchema}') - """ + AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = :param) + """, conf.dbSchema def get_field_sql(ds: CoreDatasource, conf: DatasourceConf, table_name: str = None): if ds.type == "mysql" or ds.type == "doris": - sql1 = f""" + sql1 = """ SELECT COLUMN_NAME, DATA_TYPE, @@ -142,12 +142,12 @@ def get_field_sql(ds: CoreDatasource, conf: DatasourceConf, table_name: str = No FROM INFORMATION_SCHEMA.COLUMNS WHERE - TABLE_SCHEMA = '{conf.database}' + TABLE_SCHEMA = :param1 """ - sql2 = f" AND TABLE_NAME = '{table_name}'" if table_name is not None and table_name != "" else "" - return sql1 + sql2 + sql2 = " AND TABLE_NAME = :param2" if table_name is not None and table_name != "" else "" + return sql1 + sql2, conf.database, table_name elif ds.type == "sqlServer": - sql1 = f""" + sql1 = """ SELECT COLUMN_NAME AS [COLUMN_NAME], DATA_TYPE AS [DATA_TYPE], @@ -160,12 +160,12 @@ def get_field_sql(ds: CoreDatasource, conf: DatasourceConf, table_name: str = No AND EP.minor_id = C.ORDINAL_POSITION AND EP.name = 'MS_Description' WHERE - C.TABLE_SCHEMA = '{conf.dbSchema}' + C.TABLE_SCHEMA = :param1 """ - sql2 = f" AND C.TABLE_NAME = '{table_name}'" if table_name is not None and table_name != "" else "" - return sql1 + sql2 + sql2 = " AND C.TABLE_NAME = :param2" if table_name is not None and table_name != "" else "" + return sql1 + sql2, conf.dbSchema, table_name elif ds.type == "pg" or ds.type == "excel" or ds.type == "redshift": - sql1 = f""" + sql1 = """ SELECT a.attname AS COLUMN_NAME, pg_catalog.format_type(a.atttypid, a.atttypmod) AS DATA_TYPE, col_description(c.oid, a.attnum) AS COLUMN_COMMENT @@ -174,14 +174,14 @@ def get_field_sql(ds: CoreDatasource, conf: DatasourceConf, table_name: str = No pg_catalog.pg_class c ON a.attrelid = c.oid JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace - WHERE n.nspname = '{conf.dbSchema}' + WHERE n.nspname = :param1 AND a.attnum > 0 AND NOT a.attisdropped \ """ - sql2 = f" AND c.relname = '{table_name}'" if table_name is not None and table_name != "" else "" - return sql1 + sql2 + sql2 = " AND c.relname = :param2" if table_name is not None and table_name != "" else "" + return sql1 + sql2, conf.dbSchema, table_name elif ds.type == "oracle": - sql1 = f""" + sql1 = """ SELECT col.COLUMN_NAME AS "COLUMN_NAME", (CASE @@ -201,23 +201,23 @@ def get_field_sql(ds: CoreDatasource, conf: DatasourceConf, table_name: str = No AND col.TABLE_NAME = com.TABLE_NAME AND col.COLUMN_NAME = com.COLUMN_NAME WHERE - col.OWNER = '{conf.dbSchema}' + col.OWNER = :param1 """ - sql2 = f" AND col.TABLE_NAME = '{table_name}'" if table_name is not None and table_name != "" else "" - return sql1 + sql2 + sql2 = " AND col.TABLE_NAME = :param2" if table_name is not None and table_name != "" else "" + return sql1 + sql2, conf.dbSchema, table_name elif ds.type == "ck": - sql1 = f""" + sql1 = """ SELECT name AS COLUMN_NAME, type AS DATA_TYPE, comment AS COLUMN_COMMENT FROM system.columns - WHERE database = '{conf.database}' + WHERE database = :param1 """ - sql2 = f" AND table = '{table_name}'" if table_name is not None and table_name != "" else "" - return sql1 + sql2 + sql2 = " AND table = :param2" if table_name is not None and table_name != "" else "" + return sql1 + sql2, conf.database, table_name elif ds.type == 'dm': - sql1 = f""" + sql1 = """ SELECT c.COLUMN_NAME AS "COLUMN_NAME", c.DATA_TYPE AS "DATA_TYPE", @@ -230,7 +230,7 @@ def get_field_sql(ds: CoreDatasource, conf: DatasourceConf, table_name: str = No AND c.TABLE_NAME = com.TABLE_NAME AND c.COLUMN_NAME = com.COLUMN_NAME WHERE - c.OWNER = '{conf.dbSchema}' + c.OWNER = :param1 """ - sql2 = f" AND c.TABLE_NAME = '{table_name}'" if table_name is not None and table_name != "" else "" - return sql1 + sql2 + sql2 = " AND c.TABLE_NAME = :param2" if table_name is not None and table_name != "" else "" + return sql1 + sql2, conf.dbSchema, table_name From 7405c48fd68cbe837a7955883bade493c8b565ae Mon Sep 17 00:00:00 2001 From: ulleo Date: Wed, 17 Sep 2025 15:03:08 +0800 Subject: [PATCH 040/932] feat: add SQL examples in prompt --- backend/apps/data_training/curd/data_training.py | 2 +- backend/apps/terminology/curd/terminology.py | 2 +- backend/common/core/config.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/apps/data_training/curd/data_training.py b/backend/apps/data_training/curd/data_training.py index c99b02eeb..78c533b83 100644 --- a/backend/apps/data_training/curd/data_training.py +++ b/backend/apps/data_training/curd/data_training.py @@ -198,7 +198,7 @@ def save_embeddings(session: Session, ids: List[int]): ( 1 - (embedding <=> :embedding_array) ) AS similarity FROM data_training AS child ) TEMP -WHERE similarity > {settings.EMBEDDING_SIMILARITY} and oid = :oid and datasource = :datasource +WHERE similarity > {settings.EMBEDDING_DATA_TRAINING_SIMILARITY} and oid = :oid and datasource = :datasource ORDER BY similarity DESC LIMIT {settings.EMBEDDING_DATA_TRAINING_TOP_COUNT} """ diff --git a/backend/apps/terminology/curd/terminology.py b/backend/apps/terminology/curd/terminology.py index 6e9bd4142..7d0ef1c8d 100644 --- a/backend/apps/terminology/curd/terminology.py +++ b/backend/apps/terminology/curd/terminology.py @@ -300,7 +300,7 @@ def save_embeddings(session: Session, ids: List[int]): ( 1 - (embedding <=> :embedding_array) ) AS similarity FROM terminology AS child ) TEMP -WHERE similarity > {settings.EMBEDDING_SIMILARITY} and oid = :oid +WHERE similarity > {settings.EMBEDDING_TERMINOLOGY_SIMILARITY} and oid = :oid ORDER BY similarity DESC LIMIT {settings.EMBEDDING_TERMINOLOGY_TOP_COUNT} """ diff --git a/backend/common/core/config.py b/backend/common/core/config.py index e5311307a..962a8a845 100644 --- a/backend/common/core/config.py +++ b/backend/common/core/config.py @@ -89,7 +89,9 @@ def SQLALCHEMY_DATABASE_URI(self) -> PostgresDsn | str: LOCAL_MODEL_PATH: str = '/opt/sqlbot/models' DEFAULT_EMBEDDING_MODEL: str = 'shibing624/text2vec-base-chinese' EMBEDDING_ENABLED: bool = True - EMBEDDING_SIMILARITY: float = 0.4 + EMBEDDING_DEFAULT_SIMILARITY: float = 0.4 + EMBEDDING_TERMINOLOGY_SIMILARITY: float = EMBEDDING_DEFAULT_SIMILARITY + EMBEDDING_DATA_TRAINING_SIMILARITY: float = EMBEDDING_DEFAULT_SIMILARITY EMBEDDING_DEFAULT_TOP_COUNT: int = 5 EMBEDDING_TERMINOLOGY_TOP_COUNT: int = EMBEDDING_DEFAULT_TOP_COUNT EMBEDDING_DATA_TRAINING_TOP_COUNT: int = EMBEDDING_DEFAULT_TOP_COUNT From cec71e9ed4cb2544cd94f10748c0a00524383f56 Mon Sep 17 00:00:00 2001 From: ulleo Date: Wed, 17 Sep 2025 15:13:39 +0800 Subject: [PATCH 041/932] feat: add SQL examples in prompt --- backend/apps/data_training/curd/data_training.py | 9 +++++++-- frontend/src/views/system/training/index.vue | 5 ++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/backend/apps/data_training/curd/data_training.py b/backend/apps/data_training/curd/data_training.py index 78c533b83..15260a9aa 100644 --- a/backend/apps/data_training/curd/data_training.py +++ b/backend/apps/data_training/curd/data_training.py @@ -94,7 +94,8 @@ def create_training(session: SessionDep, info: DataTrainingInfo, oid: int, trans exists = session.query( session.query(DataTraining).filter( - and_(DataTraining.question == info.question, DataTraining.oid == oid)).exists()).scalar() + and_(DataTraining.question == info.question, DataTraining.oid == oid, + DataTraining.datasource == info.datasource)).exists()).scalar() if exists: raise Exception(trans("i18n_data_training.exists_in_db")) @@ -114,8 +115,10 @@ def create_training(session: SessionDep, info: DataTrainingInfo, oid: int, trans def update_training(session: SessionDep, info: DataTrainingInfo, oid: int, trans: Trans): + if info.datasource is None: + raise Exception(trans("i18n_data_training.datasource_cannot_be_none")) + count = session.query(DataTraining).filter( - DataTraining.oid == oid, DataTraining.id == info.id ).count() if count == 0: @@ -124,6 +127,7 @@ def update_training(session: SessionDep, info: DataTrainingInfo, oid: int, trans exists = session.query( session.query(DataTraining).filter( and_(DataTraining.question == info.question, DataTraining.oid == oid, + DataTraining.datasource == info.datasource, DataTraining.id != info.id)).exists()).scalar() if exists: raise Exception(trans("i18n_data_training.exists_in_db")) @@ -131,6 +135,7 @@ def update_training(session: SessionDep, info: DataTrainingInfo, oid: int, trans stmt = update(DataTraining).where(and_(DataTraining.id == info.id)).values( question=info.question, description=info.description, + datasource=info.datasource, ) session.execute(stmt) session.commit() diff --git a/frontend/src/views/system/training/index.vue b/frontend/src/views/system/training/index.vue index fadbd961c..7c6dfac64 100644 --- a/frontend/src/views/system/training/index.vue +++ b/frontend/src/views/system/training/index.vue @@ -19,6 +19,7 @@ interface Form { id?: string | null question: string | null datasource: string | null + datasource_name: string | null description: string | null } @@ -55,6 +56,7 @@ const defaultForm = { question: null, description: null, datasource: null, + datasource_name: null, } const pageForm = ref
(cloneDeep(defaultForm)) const copyCode = () => { @@ -498,6 +500,7 @@ const onRowFormClose = () => { @@ -542,7 +545,7 @@ const onRowFormClose = () => {
- {{ pageForm.datasource }} + {{ pageForm.datasource_name }}
From d1e7c2078fea0e8fc89fca7f316027ae15ce2394 Mon Sep 17 00:00:00 2001 From: junjun Date: Wed, 17 Sep 2025 16:17:38 +0800 Subject: [PATCH 042/932] fix: Fix possible SQL injection vulnerabilities --- backend/apps/db/db.py | 8 +++---- backend/apps/db/db_sql.py | 45 ++++++++++++++++++++++++++++++++++++--- 2 files changed, 46 insertions(+), 7 deletions(-) diff --git a/backend/apps/db/db.py b/backend/apps/db/db.py index 9c1b2cfaf..66891d4f3 100644 --- a/backend/apps/db/db.py +++ b/backend/apps/db/db.py @@ -283,7 +283,7 @@ def get_tables(ds: CoreDatasource): with pymysql.connect(user=conf.username, passwd=conf.password, host=conf.host, port=conf.port, db=conf.database, connect_timeout=conf.timeout, read_timeout=conf.timeout) as conn, conn.cursor() as cursor: - cursor.execute(sql, {"param": sql_param}) + cursor.execute(sql, (sql_param,)) res = cursor.fetchall() res_list = [TableSchema(*item) for item in res] return res_list @@ -291,7 +291,7 @@ def get_tables(ds: CoreDatasource): with redshift_connector.connect(host=conf.host, port=conf.port, database=conf.database, user=conf.username, password=conf.password, timeout=conf.timeout) as conn, conn.cursor() as cursor: - cursor.execute(sql, {"param": sql_param}) + cursor.execute(sql, (sql_param,)) res = cursor.fetchall() res_list = [TableSchema(*item) for item in res] return res_list @@ -323,7 +323,7 @@ def get_fields(ds: CoreDatasource, table_name: str = None): with pymysql.connect(user=conf.username, passwd=conf.password, host=conf.host, port=conf.port, db=conf.database, connect_timeout=conf.timeout, read_timeout=conf.timeout) as conn, conn.cursor() as cursor: - cursor.execute(sql, {"param1": p1, "param2": p2}) + cursor.execute(sql, (p1, p2)) res = cursor.fetchall() res_list = [ColumnSchema(*item) for item in res] return res_list @@ -331,7 +331,7 @@ def get_fields(ds: CoreDatasource, table_name: str = None): with redshift_connector.connect(host=conf.host, port=conf.port, database=conf.database, user=conf.username, password=conf.password, timeout=conf.timeout) as conn, conn.cursor() as cursor: - cursor.execute(sql, {"param1": p1, "param2": p2}) + cursor.execute(sql, (p1, p2)) res = cursor.fetchall() res_list = [ColumnSchema(*item) for item in res] return res_list diff --git a/backend/apps/db/db_sql.py b/backend/apps/db/db_sql.py index f54837791..bcba6acc6 100644 --- a/backend/apps/db/db_sql.py +++ b/backend/apps/db/db_sql.py @@ -33,7 +33,7 @@ def get_version_sql(ds: CoreDatasource, conf: DatasourceConf): def get_table_sql(ds: CoreDatasource, conf: DatasourceConf, db_version: str = ''): - if ds.type == "mysql" or ds.type == "doris": + if ds.type == "mysql": return """ SELECT TABLE_NAME, @@ -128,8 +128,18 @@ def get_table_sql(ds: CoreDatasource, conf: DatasourceConf, db_version: str = '' pg_class WHERE relkind in ('r','p', 'f') - AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = :param) + AND relnamespace = (SELECT oid FROM pg_namespace WHERE nspname = %s) """, conf.dbSchema + elif ds.type == "doris": + return """ + SELECT + TABLE_NAME, + TABLE_COMMENT + FROM + information_schema.TABLES + WHERE + TABLE_SCHEMA = %s + """, conf.database def get_field_sql(ds: CoreDatasource, conf: DatasourceConf, table_name: str = None): @@ -164,7 +174,7 @@ def get_field_sql(ds: CoreDatasource, conf: DatasourceConf, table_name: str = No """ sql2 = " AND C.TABLE_NAME = :param2" if table_name is not None and table_name != "" else "" return sql1 + sql2, conf.dbSchema, table_name - elif ds.type == "pg" or ds.type == "excel" or ds.type == "redshift": + elif ds.type == "pg" or ds.type == "excel": sql1 = """ SELECT a.attname AS COLUMN_NAME, pg_catalog.format_type(a.atttypid, a.atttypmod) AS DATA_TYPE, @@ -180,6 +190,22 @@ def get_field_sql(ds: CoreDatasource, conf: DatasourceConf, table_name: str = No """ sql2 = " AND c.relname = :param2" if table_name is not None and table_name != "" else "" return sql1 + sql2, conf.dbSchema, table_name + elif ds.type == "redshift": + sql1 = """ + SELECT a.attname AS COLUMN_NAME, + pg_catalog.format_type(a.atttypid, a.atttypmod) AS DATA_TYPE, + col_description(c.oid, a.attnum) AS COLUMN_COMMENT + FROM pg_catalog.pg_attribute a + JOIN + pg_catalog.pg_class c ON a.attrelid = c.oid + JOIN + pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = %s + AND a.attnum > 0 + AND NOT a.attisdropped \ + """ + sql2 = " AND c.relname = %s" if table_name is not None and table_name != "" else "" + return sql1 + sql2, conf.dbSchema, table_name elif ds.type == "oracle": sql1 = """ SELECT @@ -234,3 +260,16 @@ def get_field_sql(ds: CoreDatasource, conf: DatasourceConf, table_name: str = No """ sql2 = " AND c.TABLE_NAME = :param2" if table_name is not None and table_name != "" else "" return sql1 + sql2, conf.dbSchema, table_name + elif ds.type == "doris": + sql1 = """ + SELECT + COLUMN_NAME, + DATA_TYPE, + COLUMN_COMMENT + FROM + INFORMATION_SCHEMA.COLUMNS + WHERE + TABLE_SCHEMA = %s + """ + sql2 = " AND TABLE_NAME = %s" if table_name is not None and table_name != "" else "" + return sql1 + sql2, conf.database, table_name From c3d4c642e98e6f5eb6a08facac042deb39581f72 Mon Sep 17 00:00:00 2001 From: junjun Date: Wed, 17 Sep 2025 16:22:08 +0800 Subject: [PATCH 043/932] fix: Fix possible SQL injection vulnerabilities --- backend/apps/db/db_sql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/apps/db/db_sql.py b/backend/apps/db/db_sql.py index bcba6acc6..8651e1f07 100644 --- a/backend/apps/db/db_sql.py +++ b/backend/apps/db/db_sql.py @@ -143,7 +143,7 @@ def get_table_sql(ds: CoreDatasource, conf: DatasourceConf, db_version: str = '' def get_field_sql(ds: CoreDatasource, conf: DatasourceConf, table_name: str = None): - if ds.type == "mysql" or ds.type == "doris": + if ds.type == "mysql": sql1 = """ SELECT COLUMN_NAME, From 0ccf47d05b61f5bb86cf9ec5d4a68827cb3b2d0b Mon Sep 17 00:00:00 2001 From: junjun Date: Wed, 17 Sep 2025 16:40:37 +0800 Subject: [PATCH 044/932] fix: Fix possible SQL injection vulnerabilities --- backend/apps/db/db.py | 8 ++++---- backend/apps/db/db_sql.py | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/backend/apps/db/db.py b/backend/apps/db/db.py index 66891d4f3..8d328f142 100644 --- a/backend/apps/db/db.py +++ b/backend/apps/db/db.py @@ -233,12 +233,12 @@ def get_schema(ds: CoreDatasource): with get_session(ds) as session: sql: str = '' if ds.type == "sqlServer": - sql = f"""select name from sys.schemas""" + sql = """select name from sys.schemas""" elif ds.type == "pg" or ds.type == "excel": sql = """SELECT nspname FROM pg_namespace""" elif ds.type == "oracle": - sql = f"""select * from all_users""" + sql = """select * from all_users""" with session.execute(text(sql)) as result: res = result.fetchall() res_list = [item[0] for item in res] @@ -247,7 +247,7 @@ def get_schema(ds: CoreDatasource): if ds.type == 'dm': with dmPython.connect(user=conf.username, password=conf.password, server=conf.host, port=conf.port) as conn, conn.cursor() as cursor: - cursor.execute(f"""select OBJECT_NAME from dba_objects where object_type='SCH'""", timeout=conf.timeout) + cursor.execute("""select OBJECT_NAME from dba_objects where object_type='SCH'""", timeout=conf.timeout) res = cursor.fetchall() res_list = [item[0] for item in res] return res_list @@ -255,7 +255,7 @@ def get_schema(ds: CoreDatasource): with redshift_connector.connect(host=conf.host, port=conf.port, database=conf.database, user=conf.username, password=conf.password, timeout=conf.timeout) as conn, conn.cursor() as cursor: - cursor.execute(f"""SELECT nspname FROM pg_namespace""") + cursor.execute("""SELECT nspname FROM pg_namespace""") res = cursor.fetchall() res_list = [item[0] for item in res] return res_list diff --git a/backend/apps/db/db_sql.py b/backend/apps/db/db_sql.py index 8651e1f07..030666cab 100644 --- a/backend/apps/db/db_sql.py +++ b/backend/apps/db/db_sql.py @@ -5,27 +5,27 @@ def get_version_sql(ds: CoreDatasource, conf: DatasourceConf): if ds.type == "mysql" or ds.type == "doris": - return f""" + return """ SELECT VERSION() """ elif ds.type == "sqlServer": - return f""" + return """ select SERVERPROPERTY('ProductVersion') """ elif ds.type == "pg" or ds.type == "excel": - return f""" + return """ SELECT current_setting('server_version') """ elif ds.type == "oracle": - return f""" + return """ SELECT version FROM v$instance """ elif ds.type == "ck": - return f""" + return """ select version() """ elif ds.type == 'dm': - return f""" + return """ SELECT * FROM v$version """ elif ds.type == 'redshift': From 6bc4a4f2f5c4b3879150482f3b54d281a308b674 Mon Sep 17 00:00:00 2001 From: junjun Date: Wed, 17 Sep 2025 16:57:30 +0800 Subject: [PATCH 045/932] refactor: code optimization --- backend/apps/datasource/crud/datasource.py | 3 +-- backend/apps/db/constant.py | 25 +++++++++++----------- backend/apps/db/type.py | 18 ---------------- 3 files changed, 14 insertions(+), 32 deletions(-) delete mode 100644 backend/apps/db/type.py diff --git a/backend/apps/datasource/crud/datasource.py b/backend/apps/datasource/crud/datasource.py index df55e3ab3..95f81a4d2 100644 --- a/backend/apps/datasource/crud/datasource.py +++ b/backend/apps/datasource/crud/datasource.py @@ -11,7 +11,6 @@ from apps.db.constant import DB from apps.db.db import get_tables, get_fields, exec_sql, check_connection from apps.db.engine import get_engine_config, get_engine_conn -from apps.db.type import db_type_relation from common.core.deps import SessionDep, CurrentUser, Trans from common.utils.utils import deepcopy_ignore_extra from .table import get_tables_by_ds_id @@ -70,7 +69,7 @@ def create_ds(session: SessionDep, trans: Trans, user: CurrentUser, create_ds: C ds.create_by = user.id ds.oid = user.oid if user.oid is not None else 1 ds.status = "Success" - ds.type_name = db_type_relation()[ds.type] + ds.type_name = DB.get_db(ds.type).db_name record = CoreDatasource(**ds.model_dump()) session.add(record) session.flush() diff --git a/backend/apps/db/constant.py b/backend/apps/db/constant.py index 67074ac12..4194d8b7e 100644 --- a/backend/apps/db/constant.py +++ b/backend/apps/db/constant.py @@ -13,19 +13,20 @@ def __init__(self, type_name): class DB(Enum): - mysql = ('mysql', '`', '`', ConnectType.sqlalchemy) - sqlServer = ('sqlServer', '[', ']', ConnectType.sqlalchemy) - pg = ('pg', '"', '"', ConnectType.sqlalchemy) - excel = ('excel', '"', '"', ConnectType.sqlalchemy) - oracle = ('oracle', '"', '"', ConnectType.sqlalchemy) - ck = ('ck', '"', '"', ConnectType.sqlalchemy) - dm = ('dm', '"', '"', ConnectType.py_driver) - doris = ('doris', '`', '`', ConnectType.py_driver) - redshift = ('redshift', '"', '"', ConnectType.py_driver) - es = ('es', '"', '"', ConnectType.py_driver) - - def __init__(self, type, prefix, suffix, connect_type: ConnectType): + mysql = ('mysql', 'MySQL', '`', '`', ConnectType.sqlalchemy) + sqlServer = ('sqlServer', 'Microsoft SQL Server', '[', ']', ConnectType.sqlalchemy) + pg = ('pg', 'PostgreSQL', '"', '"', ConnectType.sqlalchemy) + excel = ('excel', 'Excel/CSV', '"', '"', ConnectType.sqlalchemy) + oracle = ('oracle', 'Oracle', '"', '"', ConnectType.sqlalchemy) + ck = ('ck', 'ClickHouse', '"', '"', ConnectType.sqlalchemy) + dm = ('dm', '达梦', '"', '"', ConnectType.py_driver) + doris = ('doris', 'Apache Doris', '`', '`', ConnectType.py_driver) + redshift = ('redshift', 'AWS Redshift', '"', '"', ConnectType.py_driver) + es = ('es', 'Elasticsearch', '"', '"', ConnectType.py_driver) + + def __init__(self, type, db_name, prefix, suffix, connect_type: ConnectType): self.type = type + self.db_name = db_name self.prefix = prefix self.suffix = suffix self.connect_type = connect_type diff --git a/backend/apps/db/type.py b/backend/apps/db/type.py deleted file mode 100644 index 1e48dc654..000000000 --- a/backend/apps/db/type.py +++ /dev/null @@ -1,18 +0,0 @@ -# Author: Junjun -# Date: 2025/5/22 -from typing import Dict - - -def db_type_relation() -> Dict: - return { - "mysql": "MySQL", - "sqlServer": "Microsoft SQL Server", - "pg": "PostgreSQL", - "excel": "Excel/CSV", - "oracle": "Oracle", - "ck": "ClickHouse", - "dm": "达梦", - "doris": "Apache Doris", - "redshift": "AWS Redshift", - "es": "Elasticsearch" - } From 3ca907ed94f8ab09f17cb32269d65f270b2c4dec Mon Sep 17 00:00:00 2001 From: ulleo Date: Wed, 17 Sep 2025 17:39:50 +0800 Subject: [PATCH 046/932] fix: SQL examples search filter not work --- frontend/src/views/system/training/index.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/views/system/training/index.vue b/frontend/src/views/system/training/index.vue index 7c6dfac64..0759092bb 100644 --- a/frontend/src/views/system/training/index.vue +++ b/frontend/src/views/system/training/index.vue @@ -196,7 +196,7 @@ const search = () => { searchLoading.value = true oldKeywords.value = keywords.value trainingApi - .getList(pageInfo.currentPage, pageInfo.pageSize, { word: keywords.value }) + .getList(pageInfo.currentPage, pageInfo.pageSize, { question: keywords.value }) .then((res) => { toggleRowLoading.value = true fieldList.value = res.data From e0fa7093b5732b819338531cf37df7fd9b899d03 Mon Sep 17 00:00:00 2001 From: ulleo Date: Thu, 18 Sep 2025 10:23:17 +0800 Subject: [PATCH 047/932] feat: change SQL examples frontend text --- frontend/src/views/system/training/index.vue | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/views/system/training/index.vue b/frontend/src/views/system/training/index.vue index 0759092bb..99c81614e 100644 --- a/frontend/src/views/system/training/index.vue +++ b/frontend/src/views/system/training/index.vue @@ -228,7 +228,7 @@ const rules = { description: [ { required: true, - message: t('datasource.please_enter') + t('common.empty') + t('training.sql_statement'), + message: t('datasource.please_enter') + t('common.empty') + t('training.sample_sql'), }, ], } @@ -488,7 +488,7 @@ const onRowFormClose = () => { clearable /> - + { {{ pageForm.question }}
- +
{{ pageForm.description }}
From e267b7714d87a662469d1c680a477440b1fc1a9b Mon Sep 17 00:00:00 2001 From: ulleo Date: Thu, 18 Sep 2025 10:33:38 +0800 Subject: [PATCH 048/932] feat: change SQL examples frontend text --- frontend/src/i18n/en.json | 4 ++-- frontend/src/i18n/zh-CN.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/i18n/en.json b/frontend/src/i18n/en.json index 4bc9ff344..b306431a2 100644 --- a/frontend/src/i18n/en.json +++ b/frontend/src/i18n/en.json @@ -7,7 +7,7 @@ "AI Model Configuration": "AI Model Configuration" }, "training": { - "data_training": "Data Training", + "data_training": "SQL Sample Lib", "problem_description": "Problem Description", "sample_sql": "Sample SQL", "search_problem": "Search Problem", @@ -668,4 +668,4 @@ "setting_successfully": "Setting Successfully", "customize_theme_color": "Customize theme color" } -} \ No newline at end of file +} diff --git a/frontend/src/i18n/zh-CN.json b/frontend/src/i18n/zh-CN.json index bb597f2db..2d2611328 100644 --- a/frontend/src/i18n/zh-CN.json +++ b/frontend/src/i18n/zh-CN.json @@ -7,7 +7,7 @@ "AI Model Configuration": "模型配置" }, "training": { - "data_training": "数据训练", + "data_training": "SQL 示例库", "problem_description": "问题描述", "sample_sql": "示例 SQL", "search_problem": "搜索问题", @@ -668,4 +668,4 @@ "setting_successfully": "设置成功", "customize_theme_color": "自定义主题色" } -} \ No newline at end of file +} From a96d787cbd9e3bbfafd5595309ae642ccb79aab2 Mon Sep 17 00:00:00 2001 From: dataeaseShu Date: Thu, 18 Sep 2025 10:50:35 +0800 Subject: [PATCH 049/932] fix(Data training ): Style optimization --- frontend/src/views/system/training/index.vue | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/frontend/src/views/system/training/index.vue b/frontend/src/views/system/training/index.vue index 99c81614e..e6e790558 100644 --- a/frontend/src/views/system/training/index.vue +++ b/frontend/src/views/system/training/index.vue @@ -235,7 +235,6 @@ const rules = { const list = () => { datasourceApi.list().then((res: any) => { - console.log(res) options.value = res || [] }) } @@ -361,16 +360,16 @@ const onRowFormClose = () => { - - + + + From 919378ffc137415f95fa26cfb9c5d852abb51ab5 Mon Sep 17 00:00:00 2001 From: junjun Date: Fri, 19 Sep 2025 13:55:24 +0800 Subject: [PATCH 076/932] feat: Vector retrieval matches datasource --- backend/apps/chat/task/llm.py | 103 +++++++++--------- .../apps/datasource/embedding/ds_embedding.py | 5 +- 2 files changed, 57 insertions(+), 51 deletions(-) diff --git a/backend/apps/chat/task/llm.py b/backend/apps/chat/task/llm.py index 967bdf620..9024dfa99 100644 --- a/backend/apps/chat/task/llm.py +++ b/backend/apps/chat/task/llm.py @@ -32,6 +32,7 @@ from apps.data_training.curd.data_training import get_training_template from apps.datasource.crud.datasource import get_table_schema from apps.datasource.crud.permission import get_row_permission_filters, is_normal_user +from apps.datasource.embedding.ds_embedding import get_ds_embedding from apps.datasource.models.datasource import CoreDatasource from apps.db.db import exec_sql, get_version, check_connection from apps.system.crud.assistant import AssistantOutDs, AssistantOutDsFactory, get_assistant_ds @@ -423,61 +424,65 @@ def select_datasource(self): full_thinking_text = '' full_text = '' - json_str: Optional[str] = None if not ignore_auto_select: - _ds_list_dict = [] - for _ds in _ds_list: - _ds_list_dict.append(_ds) - datasource_msg.append( - HumanMessage(self.chat_question.datasource_user_question(orjson.dumps(_ds_list_dict).decode()))) - - self.current_logs[OperationEnum.CHOOSE_DATASOURCE] = start_log(session=self.session, - ai_modal_id=self.chat_question.ai_modal_id, - ai_modal_name=self.chat_question.ai_modal_name, - operate=OperationEnum.CHOOSE_DATASOURCE, - record_id=self.record.id, - full_message=[{'type': msg.type, - 'content': msg.content} - for - msg in datasource_msg]) - - token_usage = {} - res = self.llm.stream(datasource_msg) - for chunk in res: - SQLBotLogUtil.info(chunk) - reasoning_content_chunk = '' - if 'reasoning_content' in chunk.additional_kwargs: - reasoning_content_chunk = chunk.additional_kwargs.get('reasoning_content', '') - # else: - # reasoning_content_chunk = chunk.get('reasoning_content') - if reasoning_content_chunk is None: + if settings.EMBEDDING_ENABLED: + ds = get_ds_embedding(self.session, self.current_user, _ds_list, self.chat_question.question) + yield {'content': '{"id":' + str(ds.get('id')) + '}'} + else: + _ds_list_dict = [] + for _ds in _ds_list: + _ds_list_dict.append(_ds) + datasource_msg.append( + HumanMessage(self.chat_question.datasource_user_question(orjson.dumps(_ds_list_dict).decode()))) + + self.current_logs[OperationEnum.CHOOSE_DATASOURCE] = start_log(session=self.session, + ai_modal_id=self.chat_question.ai_modal_id, + ai_modal_name=self.chat_question.ai_modal_name, + operate=OperationEnum.CHOOSE_DATASOURCE, + record_id=self.record.id, + full_message=[{'type': msg.type, + 'content': msg.content} + for + msg in datasource_msg]) + + token_usage = {} + res = self.llm.stream(datasource_msg) + for chunk in res: + SQLBotLogUtil.info(chunk) reasoning_content_chunk = '' - full_thinking_text += reasoning_content_chunk - - full_text += chunk.content - yield {'content': chunk.content, 'reasoning_content': reasoning_content_chunk} - get_token_usage(chunk, token_usage) - datasource_msg.append(AIMessage(full_text)) - - self.current_logs[OperationEnum.CHOOSE_DATASOURCE] = end_log(session=self.session, - log=self.current_logs[ - OperationEnum.CHOOSE_DATASOURCE], - full_message=[ - {'type': msg.type, - 'content': msg.content} - for msg in datasource_msg], - reasoning_content=full_thinking_text, - token_usage=token_usage) - - json_str = extract_nested_json(full_text) - if json_str is None: - raise SingleMessageError(f'Cannot parse datasource from answer: {full_text}') + if 'reasoning_content' in chunk.additional_kwargs: + reasoning_content_chunk = chunk.additional_kwargs.get('reasoning_content', '') + # else: + # reasoning_content_chunk = chunk.get('reasoning_content') + if reasoning_content_chunk is None: + reasoning_content_chunk = '' + full_thinking_text += reasoning_content_chunk + + full_text += chunk.content + yield {'content': chunk.content, 'reasoning_content': reasoning_content_chunk} + get_token_usage(chunk, token_usage) + datasource_msg.append(AIMessage(full_text)) + + self.current_logs[OperationEnum.CHOOSE_DATASOURCE] = end_log(session=self.session, + log=self.current_logs[ + OperationEnum.CHOOSE_DATASOURCE], + full_message=[ + {'type': msg.type, + 'content': msg.content} + for msg in datasource_msg], + reasoning_content=full_thinking_text, + token_usage=token_usage) + + json_str = extract_nested_json(full_text) + if json_str is None: + raise SingleMessageError(f'Cannot parse datasource from answer: {full_text}') + ds = orjson.loads(json_str) _error: Exception | None = None _datasource: int | None = None _engine_type: str | None = None try: - data: dict = _ds_list[0] if ignore_auto_select else orjson.loads(json_str) + data: dict = _ds_list[0] if ignore_auto_select else ds if data.get('id') and data.get('id') != 0: _datasource = data['id'] @@ -516,7 +521,7 @@ def select_datasource(self): except Exception as e: _error = e - if not ignore_auto_select: + if not ignore_auto_select and not settings.EMBEDDING_ENABLED: self.record = save_select_datasource_answer(session=self.session, record_id=self.record.id, answer=orjson.dumps({'content': full_text}).decode(), datasource=_datasource, diff --git a/backend/apps/datasource/embedding/ds_embedding.py b/backend/apps/datasource/embedding/ds_embedding.py index f0524b763..48ef07ef2 100644 --- a/backend/apps/datasource/embedding/ds_embedding.py +++ b/backend/apps/datasource/embedding/ds_embedding.py @@ -5,7 +5,8 @@ import traceback from apps.ai_model.embedding import EmbeddingModelCache -from apps.datasource.crud.datasource import get_table_schema, get_ds +from apps.datasource.crud.datasource import get_table_schema +from apps.datasource.models.datasource import CoreDatasource from common.core.deps import SessionDep, CurrentUser @@ -28,7 +29,7 @@ def get_ds_embedding(session: SessionDep, current_user: CurrentUser, _ds_list, q _list = [] for _ds in _ds_list: if _ds.get('id'): - ds = get_ds(session, _ds.get('id')) + ds = session.get(CoreDatasource, _ds.get('id')) table_schema = get_table_schema(session, current_user, ds) ds_info = f"{ds.name}, {ds.description}\n" From 336ffcf2a11f550579b065db0801929eb99103ee Mon Sep 17 00:00:00 2001 From: junjun Date: Fri, 19 Sep 2025 14:49:04 +0800 Subject: [PATCH 077/932] feat: Vector retrieval matches datasource --- backend/apps/datasource/embedding/ds_embedding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/apps/datasource/embedding/ds_embedding.py b/backend/apps/datasource/embedding/ds_embedding.py index 48ef07ef2..afa1ec693 100644 --- a/backend/apps/datasource/embedding/ds_embedding.py +++ b/backend/apps/datasource/embedding/ds_embedding.py @@ -50,7 +50,7 @@ def get_ds_embedding(session: SessionDep, current_user: CurrentUser, _ds_list, q _list[index]['cosine_similarity'] = cosine_similarity(q_embedding, item) _list.sort(key=lambda x: x['cosine_similarity'], reverse=True) - print(json.dumps(_list)) + print(len(_list)) ds = _list[0].get('ds') return {"id": ds.id, "name": ds.name, "description": ds.description} except Exception: From eebb601a6dd854733823c17010eb6860465b4a46 Mon Sep 17 00:00:00 2001 From: junjun Date: Fri, 19 Sep 2025 15:04:37 +0800 Subject: [PATCH 078/932] feat: Vector retrieval matches datasource --- backend/apps/datasource/embedding/ds_embedding.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/backend/apps/datasource/embedding/ds_embedding.py b/backend/apps/datasource/embedding/ds_embedding.py index afa1ec693..2b637e181 100644 --- a/backend/apps/datasource/embedding/ds_embedding.py +++ b/backend/apps/datasource/embedding/ds_embedding.py @@ -8,6 +8,7 @@ from apps.datasource.crud.datasource import get_table_schema from apps.datasource.models.datasource import CoreDatasource from common.core.deps import SessionDep, CurrentUser +from common.utils.utils import SQLBotLogUtil def cosine_similarity(vec_a, vec_b): @@ -50,7 +51,10 @@ def get_ds_embedding(session: SessionDep, current_user: CurrentUser, _ds_list, q _list[index]['cosine_similarity'] = cosine_similarity(q_embedding, item) _list.sort(key=lambda x: x['cosine_similarity'], reverse=True) - print(len(_list)) + # print(len(_list)) + SQLBotLogUtil.info(json.dumps( + [{"id": ele.get("id"), "name": ele.get("name"), "cosine_similarity": ele.get("cosine_similarity")} + for ele in _list])) ds = _list[0].get('ds') return {"id": ds.id, "name": ds.name, "description": ds.description} except Exception: From 632fad4dec91cb9c96caecbe0f952c6e90c12a58 Mon Sep 17 00:00:00 2001 From: junjun Date: Fri, 19 Sep 2025 15:16:49 +0800 Subject: [PATCH 079/932] feat: Vector retrieval matches datasource --- backend/apps/datasource/embedding/ds_embedding.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/apps/datasource/embedding/ds_embedding.py b/backend/apps/datasource/embedding/ds_embedding.py index 2b637e181..a8875744e 100644 --- a/backend/apps/datasource/embedding/ds_embedding.py +++ b/backend/apps/datasource/embedding/ds_embedding.py @@ -53,7 +53,7 @@ def get_ds_embedding(session: SessionDep, current_user: CurrentUser, _ds_list, q _list.sort(key=lambda x: x['cosine_similarity'], reverse=True) # print(len(_list)) SQLBotLogUtil.info(json.dumps( - [{"id": ele.get("id"), "name": ele.get("name"), "cosine_similarity": ele.get("cosine_similarity")} + [{"id": ele.get("id"), "name": ele.get("ds").name, "cosine_similarity": ele.get("cosine_similarity")} for ele in _list])) ds = _list[0].get('ds') return {"id": ds.id, "name": ds.name, "description": ds.description} From b566e695adaae357fe79e46760f4e7c24b10a8d8 Mon Sep 17 00:00:00 2001 From: ulleo Date: Fri, 19 Sep 2025 15:52:11 +0800 Subject: [PATCH 080/932] fix: improve export empty chat excel error info --- backend/apps/chat/api/chat.py | 12 ++++++--- backend/locales/en.json | 3 +++ backend/locales/zh-CN.json | 3 +++ frontend/src/api/chat.ts | 6 ++++- .../src/views/chat/chat-block/ChartBlock.vue | 25 +++++++++++++++++++ 5 files changed, 45 insertions(+), 4 deletions(-) diff --git a/backend/apps/chat/api/chat.py b/backend/apps/chat/api/chat.py index de53fd269..00f7def02 100644 --- a/backend/apps/chat/api/chat.py +++ b/backend/apps/chat/api/chat.py @@ -13,7 +13,7 @@ delete_chat, get_chat_chart_data, get_chat_predict_data, get_chat_with_records_with_data, get_chat_record_by_id from apps.chat.models.chat_model import CreateChat, ChatRecord, RenameChat, ChatQuestion, ExcelData from apps.chat.task.llm import LLMService -from common.core.deps import CurrentAssistant, SessionDep, CurrentUser +from common.core.deps import CurrentAssistant, SessionDep, CurrentUser, Trans router = APIRouter(tags=["Data Q&A"], prefix="/chat") @@ -106,7 +106,7 @@ async def start_chat(session: SessionDep, current_user: CurrentUser): async def recommend_questions(session: SessionDep, current_user: CurrentUser, chat_record_id: int, current_assistant: CurrentAssistant): def _return_empty(): - yield 'data:' + orjson.dumps( {'content': '[]', 'type': 'recommended_question'}).decode() + '\n\n' + yield 'data:' + orjson.dumps({'content': '[]', 'type': 'recommended_question'}).decode() + '\n\n' try: record = get_chat_record_by_id(session, chat_record_id) @@ -201,10 +201,16 @@ def _err(_e: Exception): @router.post("/excel/export") -async def export_excel(excel_data: ExcelData): +async def export_excel(excel_data: ExcelData, trans: Trans): def inner(): _fields_list = [] data = [] + if not excel_data.data: + raise HTTPException( + status_code=500, + detail=trans("i18n_excel_export.data_is_empty") + ) + for _data in excel_data.data: _row = [] for field in excel_data.axis: diff --git a/backend/locales/en.json b/backend/locales/en.json index 52fe00a57..df427789a 100644 --- a/backend/locales/en.json +++ b/backend/locales/en.json @@ -47,5 +47,8 @@ "datasource_cannot_be_none": "Datasource cannot be none", "data_training_not_exists": "Example does not exists", "exists_in_db": "Question exists" + }, + "i18n_excel_export": { + "data_is_empty": "The form data is empty, cannot export data" } } \ No newline at end of file diff --git a/backend/locales/zh-CN.json b/backend/locales/zh-CN.json index 812055e77..63961877e 100644 --- a/backend/locales/zh-CN.json +++ b/backend/locales/zh-CN.json @@ -47,5 +47,8 @@ "datasource_cannot_be_none": "数据源不能为空", "data_training_not_exists": "该示例不存在", "exists_in_db": "该问题已存在" + }, + "i18n_excel_export": { + "data_is_empty": "表单数据为空，无法导出数据" } } \ No newline at end of file diff --git a/frontend/src/api/chat.ts b/frontend/src/api/chat.ts index c1b526c78..76c436ddb 100644 --- a/frontend/src/api/chat.ts +++ b/frontend/src/api/chat.ts @@ -332,5 +332,9 @@ export const chatApi = { return request.fetchStream(`/chat/recommend_questions/${record_id}`, {}, controller) }, checkLLMModel: () => request.get('/system/aimodel/default', { requestOptions: { silent: true } }), - export2Excel: (data: any) => request.post('/chat/excel/export', data, { responseType: 'blob' }), + export2Excel: (data: any) => + request.post('/chat/excel/export', data, { + responseType: 'blob', + requestOptions: { customError: true }, + }), } diff --git a/frontend/src/views/chat/chat-block/ChartBlock.vue b/frontend/src/views/chat/chat-block/ChartBlock.vue index a8d8c502d..6220fb426 100644 --- a/frontend/src/views/chat/chat-block/ChartBlock.vue +++ b/frontend/src/views/chat/chat-block/ChartBlock.vue @@ -255,6 +255,31 @@ function exportToExcel() { link.click() document.body.removeChild(link) }) + .catch(async (error) => { + if (error.response) { + try { + let text = await error.response.data.text() + try { + text = JSON.parse(text) + } finally { + ElMessage({ + message: text, + type: 'error', + showClose: true, + }) + } + } catch (e) { + console.error('Error processing error response:', e) + } + } else { + console.error('Other error:', error) + ElMessage({ + message: error, + type: 'error', + showClose: true, + }) + } + }) .finally(() => { loading.value = false }) From 51c419df32fc1a365a73d8e49ce16a7baf007ff7 Mon Sep 17 00:00:00 2001 From: dataeaseShu Date: Fri, 19 Sep 2025 16:20:03 +0800 Subject: [PATCH 081/932] feat(Smart Question): In the conversation window, when there are many messages and a scroll bar appears, the scroll bar will automatically scroll to the bottom --- frontend/src/views/chat/index.vue | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/frontend/src/views/chat/index.vue b/frontend/src/views/chat/index.vue index 8f2e5bfe0..afb99444e 100644 --- a/frontend/src/views/chat/index.vue +++ b/frontend/src/views/chat/index.vue @@ -501,7 +501,7 @@ let scrollTopVal = 0 let scrolling = false const scrollBottom = () => { if (scrolling) return - if (!isTyping.value) { + if (!isTyping.value && !getRecommendQuestionsLoading.value) { clearInterval(scrollTime) } chatListRef.value!.setScrollTop(innerRef.value!.clientHeight) @@ -671,11 +671,14 @@ function quickAsk(question: string) { } const chartAnswerRef = ref() - +const getRecommendQuestionsLoading = ref(false) async function onChartAnswerFinish(id: number) { + getRecommendQuestionsLoading.value = true loading.value = false isTyping.value = false - getRecommendQuestions(id) + getRecommendQuestions(id).finally(() => { + getRecommendQuestionsLoading.value = false + }) } function onChartAnswerError() { From 97ffc34a422cb8b886aa09f3e57740f3bda1dd7d Mon Sep 17 00:00:00 2001 From: dataeaseShu Date: Fri, 19 Sep 2025 16:36:59 +0800 Subject: [PATCH 082/932] fix(Smart Question): In the conversation window, when there are many messages and a scroll bar appears, the scroll bar will automatically scroll to the bottom --- frontend/src/views/chat/RecommendQuestion.vue | 3 ++- frontend/src/views/chat/index.vue | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/frontend/src/views/chat/RecommendQuestion.vue b/frontend/src/views/chat/RecommendQuestion.vue index a192b0a43..5acb9012c 100644 --- a/frontend/src/views/chat/RecommendQuestion.vue +++ b/frontend/src/views/chat/RecommendQuestion.vue @@ -21,7 +21,7 @@ const props = withDefaults( } ) -const emits = defineEmits(['clickQuestion', 'update:currentChat', 'stop']) +const emits = defineEmits(['clickQuestion', 'update:currentChat', 'stop', 'loadingOver']) const loading = ref(false) @@ -134,6 +134,7 @@ async function getRecommendQuestions() { } } finally { loading.value = false + emits('loadingOver') } } diff --git a/frontend/src/views/chat/index.vue b/frontend/src/views/chat/index.vue index afb99444e..60accc1c5 100644 --- a/frontend/src/views/chat/index.vue +++ b/frontend/src/views/chat/index.vue @@ -195,6 +195,7 @@ :first-chat="message.first_chat" @click-question="quickAsk" @stop="onChatStop" + @loadingOver="loadingOver" /> @@ -676,9 +678,11 @@ async function onChartAnswerFinish(id: number) { getRecommendQuestionsLoading.value = true loading.value = false isTyping.value = false - getRecommendQuestions(id).finally(() => { - getRecommendQuestionsLoading.value = false - }) + getRecommendQuestions(id) +} + +const loadingOver = () => { + getRecommendQuestionsLoading.value = false } function onChartAnswerError() { From 313e1410c4b973d7b75055b94a075c20fd558a43 Mon Sep 17 00:00:00 2001 From: ulleo Date: Fri, 19 Sep 2025 17:03:15 +0800 Subject: [PATCH 083/932] fix: MCP chat run error while SQL execution result is empty --- backend/apps/chat/task/llm.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/backend/apps/chat/task/llm.py b/backend/apps/chat/task/llm.py index 9024dfa99..5f9881990 100644 --- a/backend/apps/chat/task/llm.py +++ b/backend/apps/chat/task/llm.py @@ -1103,11 +1103,14 @@ def run_task(self, in_chat: bool = True): _fields_list.append(field if not _fields.get(field) else _fields.get(field)) data.append(_row) _fields_skip = True - df = pd.DataFrame(np.array(data), columns=_fields_list) - markdown_table = df.to_markdown(index=False) - yield markdown_table + '\n\n' - record = self.finish() + if not data or not _fields_list: + yield 'The SQL execution result is empty.\n\n' + else: + df = pd.DataFrame(np.array(data), columns=_fields_list) + markdown_table = df.to_markdown(index=False) + yield markdown_table + '\n\n' + if in_chat: yield 'data:' + orjson.dumps({'type': 'finish'}).decode() + '\n\n' else: @@ -1135,6 +1138,8 @@ def run_task(self, in_chat: bool = True): yield 'data:' + orjson.dumps({'content': error_msg, 'type': 'error'}).decode() + '\n\n' else: yield f'> ❌ **ERROR**\n\n> \n\n> {error_msg}。' + finally: + self.finish() def run_recommend_questions_task_async(self): self.future = executor.submit(self.run_recommend_questions_task_cache) From 22d989745ef43ca60d8d2fe824945480f206cb4f Mon Sep 17 00:00:00 2001 From: maninhill <41712985+maninhill@users.noreply.github.com> Date: Fri, 19 Sep 2025 20:27:20 +0800 Subject: [PATCH 084/932] Add Cordys CRM to the list of projects --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 95b224b79..c9768e394 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ docker run -d \ - [1Panel](https://github.com/1panel-dev/1panel/) - 现代化、开源的 Linux 服务器运维管理面板 - [MaxKB](https://github.com/1panel-dev/MaxKB/) - 强大易用的企业级智能体平台 - [JumpServer](https://github.com/jumpserver/jumpserver/) - 广受欢迎的开源堡垒机 +- [Cordys CRM](https://github.com/1Panel-dev/CordysCRM) - 新一代的开源 AI CRM 系统 - [Halo](https://github.com/halo-dev/halo/) - 强大易用的开源建站工具 - [MeterSphere](https://github.com/metersphere/metersphere/) - 新一代的开源持续测试工具 From 69550a89a1b08e8cb7c910ca5d7d9f3a64a5acad Mon Sep 17 00:00:00 2001 From: fit2cloud-chenyw Date: Mon, 22 Sep 2025 10:15:09 +0800 Subject: [PATCH 085/932] perf: Remove non-essential AES IV fields --- frontend/src/views/system/embedded/iframe.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/views/system/embedded/iframe.vue b/frontend/src/views/system/embedded/iframe.vue index e35f9da01..a12d7ebce 100644 --- a/frontend/src/views/system/embedded/iframe.vue +++ b/frontend/src/views/system/embedded/iframe.vue @@ -784,7 +784,7 @@ const saveHandler = () => { autocomplete="off" /> - +