From 2388d0371340700125b094c187ca7f1b838c6bd0 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Tue, 12 May 2026 14:22:45 +0300 Subject: [PATCH 01/20] Replaces erlang's record tuple w/ json as search vector source --- apps/dmt/src/dmt_repository.erl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/apps/dmt/src/dmt_repository.erl b/apps/dmt/src/dmt_repository.erl index 4fa58ce..c35ad4d 100644 --- a/apps/dmt/src/dmt_repository.erl +++ b/apps/dmt/src/dmt_repository.erl @@ -618,7 +618,16 @@ get_new_version(Worker, AuthorID) -> insert_object(Worker, Type, ID0, Version, Data0) -> ID1 = dmt_mapper:ref_to_string(ID0), Data1 = dmt_mapper:object_to_string(Data0), - SearchVector = dmt_mapper:extract_searchable_text_from_term(Data0), + %% NOTE Turns string containing nested json into space-separated string of + %% words/lexems. + %% As example, this turns + %% { + %% "hello": "world", + %% "test": 42 + %% } + %% into + %% hello world test 42 + SearchVector = dmt_mapper:extract_searchable_text_from_term(jsx:decode(Data1)), case dmt_database:insert_object(Worker, ID1, Type, Version, Data1, SearchVector) of ok -> From 64533124dbcd801ba69098fe3c483b6577429b4b Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Tue, 30 Jun 2026 17:14:23 +0300 Subject: [PATCH 02/20] XYZ-250: Adds JSONish search vector and storage migrations --- apps/dmt/src/dmt_repository.erl | 32 +++++++------ .../dmt_repository_client_tests_SUITE.erl | 2 +- .../test/dmt_search_objects_tests_SUITE.erl | 2 +- elvis.config | 2 +- ...2722010-create-search-vector-tmp-table.sql | 6 +++ ...2011-fill-search-vector-fill-tmp-table.erl | 46 +++++++++++++++++++ ...-entities-with-search-vector-tmp-table.sql | 4 ++ ...782722013-drop-search-vector-tmp-table.sql | 1 + rebar.config | 1 + 9 files changed, 78 insertions(+), 18 deletions(-) create mode 100644 migrations/1782722010-create-search-vector-tmp-table.sql create mode 100644 migrations/1782722011-fill-search-vector-fill-tmp-table.erl create mode 100644 migrations/1782722012-update-entities-with-search-vector-tmp-table.sql create mode 100644 migrations/1782722013-drop-search-vector-tmp-table.sql diff --git a/apps/dmt/src/dmt_repository.erl b/apps/dmt/src/dmt_repository.erl index 4578305..3de4d62 100644 --- a/apps/dmt/src/dmt_repository.erl +++ b/apps/dmt/src/dmt_repository.erl @@ -753,19 +753,8 @@ get_new_version(Worker, AuthorID) -> ) -> object_ref() | no_return(). insert_object(Worker, Type, ID0, Version, Data0) -> + {ok, Data1, SearchVector} = object_to_serialized(Data0), ID1 = dmt_mapper:ref_to_string(ID0), - Data1 = dmt_mapper:object_to_string(Data0), - %% NOTE Turns string containing nested json into space-separated string of - %% words/lexems. - %% As example, this turns - %% { - %% "hello": "world", - %% "test": 42 - %% } - %% into - %% hello world test 42 - SearchVector = dmt_mapper:extract_searchable_text_from_term(jsx:decode(Data1)), - case dmt_database:insert_object(Worker, ID1, Type, Version, Data1, SearchVector) of ok -> ID0; @@ -807,15 +796,28 @@ get_object_field({_, _, _, ref, _}, _Data, {_Type, Ref}) -> get_object_field({_, _, _, data, _}, Data, _Ref) -> Data. +-spec object_to_serialized(domain_object()) -> {ok, JsonBinary :: binary(), SearchString :: string()}. +object_to_serialized(Data0) -> + Data1 = dmt_mapper:object_to_string(Data0), + %% NOTE Turns string containing nested json into space-separated string of + %% words/lexems. + %% As example, this turns + %% { + %% "hello": "world", + %% "test": 42 + %% } + %% into + %% hello world test 42 + SearchVector = dmt_mapper:extract_searchable_text_from_term(jsx:decode(Data1)), + {ok, Data1, SearchVector}. + -spec update_object( dmt_database:worker(), dmt_database:entity_type(), object_ref(), boolean(), domain_object(), dmt_object:version() ) -> ok | no_return(). update_object(Worker, Type, ID0, IsActive, Data0, Version) -> - Data1 = dmt_mapper:object_to_string(Data0), + {ok, Data1, SearchVector} = object_to_serialized(Data0), ID1 = dmt_mapper:ref_to_string(ID0), - SearchVector = dmt_mapper:extract_searchable_text_from_term(Data0), - case dmt_database:update_object( Worker, diff --git a/apps/dmt/test/dmt_repository_client_tests_SUITE.erl b/apps/dmt/test/dmt_repository_client_tests_SUITE.erl index 53c66ac..73d0b28 100644 --- a/apps/dmt/test/dmt_repository_client_tests_SUITE.erl +++ b/apps/dmt/test/dmt_repository_client_tests_SUITE.erl @@ -36,7 +36,7 @@ %% Initialize per suite init_per_suite(Config) -> - {Apps, _Ret} = dmt_ct_helper:start_apps([woody, scoper, epg_connector, dmt]), + {Apps, _Ret} = dmt_ct_helper:start_apps([woody, scoper, epg_connector, brod, dmt]), ApiClient = dmt_ct_helper:create_client(), [{client, ApiClient}, {apps, Apps} | Config]. diff --git a/apps/dmt/test/dmt_search_objects_tests_SUITE.erl b/apps/dmt/test/dmt_search_objects_tests_SUITE.erl index 8e6ce2b..d0a50a4 100644 --- a/apps/dmt/test/dmt_search_objects_tests_SUITE.erl +++ b/apps/dmt/test/dmt_search_objects_tests_SUITE.erl @@ -42,7 +42,7 @@ %% Initialize per suite init_per_suite(Config) -> - {Apps, _Ret} = dmt_ct_helper:start_apps([woody, scoper, epg_connector, dmt]), + {Apps, _Ret} = dmt_ct_helper:start_apps([woody, scoper, epg_connector, brod, dmt]), ApiClient = dmt_ct_helper:create_client(), [{client, ApiClient}, {apps, Apps} | Config]. diff --git a/elvis.config b/elvis.config index 2815340..e61cc07 100644 --- a/elvis.config +++ b/elvis.config @@ -2,7 +2,7 @@ {elvis, [ {config, [ #{ - dirs => ["apps/**/src", "apps/**/include"], + dirs => ["migrations/*.erl", "apps/**/src", "apps/**/include"], filter => "*.erl", ruleset => erl_files, rules => [ diff --git a/migrations/1782722010-create-search-vector-tmp-table.sql b/migrations/1782722010-create-search-vector-tmp-table.sql new file mode 100644 index 0000000..d2ea64e --- /dev/null +++ b/migrations/1782722010-create-search-vector-tmp-table.sql @@ -0,0 +1,6 @@ +CREATE TABLE tmp_entity_search_vector ( + id TEXT NOT NULL, + version BIGINT NOT NULL REFERENCES version(version), + search_vector tsvector, + PRIMARY KEY (id, version) +); diff --git a/migrations/1782722011-fill-search-vector-fill-tmp-table.erl b/migrations/1782722011-fill-search-vector-fill-tmp-table.erl new file mode 100644 index 0000000..53e21af --- /dev/null +++ b/migrations/1782722011-fill-search-vector-fill-tmp-table.erl @@ -0,0 +1,46 @@ +-module('1782722011-fill-search-vector-fill-tmp-table'). + +-export([perform/2]). + +-define(BATCH_SIZE, 100). + +perform(Conn, _MigrationOpts) -> + ObjectsCount = count_objects(Conn), + BatchesCount = round(math:ceil(ObjectsCount / ?BATCH_SIZE)), + ok = lists:foreach( + fun(N) -> + Objects0 = collect_objects(Conn, Offset, ?BATCH_SIZE), + Objects1 = lists:map(fun form_search_vector/1, Objects0), + insert_objects_into_buffer(Conn, Objects1) + end, + lists:seq(0, BatchesCount - 1) + ). + +%% + +count_objects(Conn) -> + {ok, _Cols, Count} = epgsql:squery(Conn, "SELECT COUNT(*) FROM entity"), + binary_to_integer(Count). + +collect_objects(Conn, Offset, Limit) -> + Query = """ + SELECT id, version, entity_type, data + FROM entity + ORDER BY id, version ASC + LIMIT $1 OFFSET $2 + """, + {ok, _Cols, Objects} = epgsql:equery(Conn, Query, [Limit, Offset]), + Objects. + +form_search_vector({ID, Version, _Type, Data}) -> + SearchVector = dmt_mapper:extract_searchable_text_from_term(jsx:decode(Data)), + [ID, Version, SearchVector]. + +insert_objects_into_buffer(Conn, Objects) -> + epgsql:execute_batch( + """ + INSERT INTO tmp_entity_search_vector (id, version, search_vector) + VALUES ($1, $2, to_tsvector('multilingual', $3)) + """, + Objects + ). diff --git a/migrations/1782722012-update-entities-with-search-vector-tmp-table.sql b/migrations/1782722012-update-entities-with-search-vector-tmp-table.sql new file mode 100644 index 0000000..fbf9375 --- /dev/null +++ b/migrations/1782722012-update-entities-with-search-vector-tmp-table.sql @@ -0,0 +1,4 @@ +UPDATE entity AS entity +SET entity.search_vector = buffer.search_vector +FROM (SELECT id, version, search_vector FROM tmp_entity_search_vector) AS buffer +WHERE entity.id = buffer.id AND entity.version = buffer.version; diff --git a/migrations/1782722013-drop-search-vector-tmp-table.sql b/migrations/1782722013-drop-search-vector-tmp-table.sql new file mode 100644 index 0000000..2290f96 --- /dev/null +++ b/migrations/1782722013-drop-search-vector-tmp-table.sql @@ -0,0 +1 @@ +DROP TABLE tmp_entity_search_vector; diff --git a/rebar.config b/rebar.config index 36d20c7..1f52e8b 100644 --- a/rebar.config +++ b/rebar.config @@ -148,6 +148,7 @@ {print_width, 120}, {files, [ "apps/dmt*/{src,include,test}/*.{hrl,erl,app.src}", + "migrations/*.erl", "rebar.config", "elvis.config", "config/sys.config", From 96fabae9a5b8e303c7345bbddde51925f9532b5d Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Tue, 30 Jun 2026 17:48:31 +0300 Subject: [PATCH 03/20] Fixes migration's `badarg` when counting objects --- ...2011-fill-search-vector-fill-tmp-table.erl | 35 ++++++++++++------- ...-entities-with-search-vector-tmp-table.sql | 8 ++--- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/migrations/1782722011-fill-search-vector-fill-tmp-table.erl b/migrations/1782722011-fill-search-vector-fill-tmp-table.erl index 53e21af..63f5329 100644 --- a/migrations/1782722011-fill-search-vector-fill-tmp-table.erl +++ b/migrations/1782722011-fill-search-vector-fill-tmp-table.erl @@ -7,9 +7,9 @@ perform(Conn, _MigrationOpts) -> ObjectsCount = count_objects(Conn), BatchesCount = round(math:ceil(ObjectsCount / ?BATCH_SIZE)), - ok = lists:foreach( + lists:foreach( fun(N) -> - Objects0 = collect_objects(Conn, Offset, ?BATCH_SIZE), + Objects0 = collect_objects(Conn, N * ?BATCH_SIZE, ?BATCH_SIZE), Objects1 = lists:map(fun form_search_vector/1, Objects0), insert_objects_into_buffer(Conn, Objects1) end, @@ -19,8 +19,10 @@ perform(Conn, _MigrationOpts) -> %% count_objects(Conn) -> - {ok, _Cols, Count} = epgsql:squery(Conn, "SELECT COUNT(*) FROM entity"), - binary_to_integer(Count). + case epgsql:squery(Conn, "SELECT COUNT(*) FROM entity") of + {ok, _Cols, [{Count}]} -> binary_to_integer(Count); + {error, Error} -> erlang:throw(Error) + end. collect_objects(Conn, Offset, Limit) -> Query = """ @@ -29,18 +31,25 @@ collect_objects(Conn, Offset, Limit) -> ORDER BY id, version ASC LIMIT $1 OFFSET $2 """, - {ok, _Cols, Objects} = epgsql:equery(Conn, Query, [Limit, Offset]), - Objects. + case epgsql:equery(Conn, Query, [Limit, Offset]) of + {ok, _Cols, Objects} -> Objects; + {error, Error} -> erlang:throw(Error) + end. form_search_vector({ID, Version, _Type, Data}) -> SearchVector = dmt_mapper:extract_searchable_text_from_term(jsx:decode(Data)), [ID, Version, SearchVector]. insert_objects_into_buffer(Conn, Objects) -> - epgsql:execute_batch( - """ - INSERT INTO tmp_entity_search_vector (id, version, search_vector) - VALUES ($1, $2, to_tsvector('multilingual', $3)) - """, - Objects - ). + case + epgsql:execute_batch( + """ + INSERT INTO tmp_entity_search_vector (id, version, search_vector) + VALUES ($1, $2, to_tsvector('multilingual', $3)) + """, + Objects + ) + of + {error, Error} -> erlang:throw(Error); + _ -> ok + end. diff --git a/migrations/1782722012-update-entities-with-search-vector-tmp-table.sql b/migrations/1782722012-update-entities-with-search-vector-tmp-table.sql index fbf9375..1205760 100644 --- a/migrations/1782722012-update-entities-with-search-vector-tmp-table.sql +++ b/migrations/1782722012-update-entities-with-search-vector-tmp-table.sql @@ -1,4 +1,4 @@ -UPDATE entity AS entity -SET entity.search_vector = buffer.search_vector -FROM (SELECT id, version, search_vector FROM tmp_entity_search_vector) AS buffer -WHERE entity.id = buffer.id AND entity.version = buffer.version; +UPDATE entity +SET search_vector = tmp.search_vector +FROM (SELECT id, version, search_vector FROM tmp_entity_search_vector) AS tmp +WHERE entity.id = tmp.id AND entity.version = tmp.version; From 9212c4f0f84c31fcf33323080be302eefd5b715e Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Wed, 1 Jul 2026 15:40:08 +0300 Subject: [PATCH 04/20] Refactors filling migration into bulk insert w/ prepared statement --- ...2011-fill-search-vector-fill-tmp-table.erl | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/migrations/1782722011-fill-search-vector-fill-tmp-table.erl b/migrations/1782722011-fill-search-vector-fill-tmp-table.erl index 63f5329..c7cc80b 100644 --- a/migrations/1782722011-fill-search-vector-fill-tmp-table.erl +++ b/migrations/1782722011-fill-search-vector-fill-tmp-table.erl @@ -2,7 +2,7 @@ -export([perform/2]). --define(BATCH_SIZE, 100). +-define(BATCH_SIZE, 500). perform(Conn, _MigrationOpts) -> ObjectsCount = count_objects(Conn), @@ -40,16 +40,30 @@ form_search_vector({ID, Version, _Type, Data}) -> SearchVector = dmt_mapper:extract_searchable_text_from_term(jsx:decode(Data)), [ID, Version, SearchVector]. +insert_objects_into_buffer(_Conn, []) -> + ok; insert_objects_into_buffer(Conn, Objects) -> - case - epgsql:execute_batch( - """ - INSERT INTO tmp_entity_search_vector (id, version, search_vector) - VALUES ($1, $2, to_tsvector('multilingual', $3)) - """, - Objects + QueryHead = """ + INSERT INTO tmp_entity_search_vector (id, version, search_vector) + VALUES + + """, + Values = lists:join( + $,, + lists:map( + fun({I, [ID, Version, SearchVector]}) -> + PH = [ph(I, 1), ph(I, 2), ["to_tsvector('multilingual',", ph(I, 3), ")"]], + [$(, lists:join($,, PH), $)] + end, + lists:zip(lists:seq(1, length(Objects)), Objects) ) - of + ), + Params = lists:append(Objects), + case epgsql:equery(Conn, [QueryHead | Values], Params) of {error, Error} -> erlang:throw(Error); _ -> ok end. + +%% Placeholder helper +ph(I, N) -> + [$$, integer_to_binary((I - 1) * 3 + N)]. From 3fb109d269ac4bf599107d464cdb0a1fc82a58af Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Wed, 1 Jul 2026 15:49:15 +0300 Subject: [PATCH 05/20] Consolidates migration commands into single module --- apps/dmt/src/dmt_sup.erl | 6 +-- ...2722010-create-search-vector-tmp-table.sql | 6 --- ...2011-fill-search-vector-fill-tmp-table.erl | 42 ++++++++++++++++++- ...-entities-with-search-vector-tmp-table.sql | 4 -- ...782722013-drop-search-vector-tmp-table.sql | 1 - 5 files changed, 43 insertions(+), 16 deletions(-) delete mode 100644 migrations/1782722010-create-search-vector-tmp-table.sql delete mode 100644 migrations/1782722012-update-entities-with-search-vector-tmp-table.sql delete mode 100644 migrations/1782722013-drop-search-vector-tmp-table.sql diff --git a/apps/dmt/src/dmt_sup.erl b/apps/dmt/src/dmt_sup.erl index 9f02c07..b34b880 100644 --- a/apps/dmt/src/dmt_sup.erl +++ b/apps/dmt/src/dmt_sup.erl @@ -67,10 +67,10 @@ dbinit() -> MigrationsPath = WorkDir ++ "/migrations", case dmt_db_migration:run(MigrationsPath) of ok -> - _ = logger:warning("entity_type: ~p", [ - epg_pool:query(default_pool, "SELECT * FROM entity_type;") - ]), ok; + %% TODO Pass through stacktrace to make migration debug easier. + %% Also consider reraise option in migrator transaction and use separate + %% transactions per migration. {error, Reason} -> throw({migrations_error, Reason}) end. diff --git a/migrations/1782722010-create-search-vector-tmp-table.sql b/migrations/1782722010-create-search-vector-tmp-table.sql deleted file mode 100644 index d2ea64e..0000000 --- a/migrations/1782722010-create-search-vector-tmp-table.sql +++ /dev/null @@ -1,6 +0,0 @@ -CREATE TABLE tmp_entity_search_vector ( - id TEXT NOT NULL, - version BIGINT NOT NULL REFERENCES version(version), - search_vector tsvector, - PRIMARY KEY (id, version) -); diff --git a/migrations/1782722011-fill-search-vector-fill-tmp-table.erl b/migrations/1782722011-fill-search-vector-fill-tmp-table.erl index c7cc80b..9e93177 100644 --- a/migrations/1782722011-fill-search-vector-fill-tmp-table.erl +++ b/migrations/1782722011-fill-search-vector-fill-tmp-table.erl @@ -5,19 +5,57 @@ -define(BATCH_SIZE, 500). perform(Conn, _MigrationOpts) -> + ok = create_tmp_table(Conn), ObjectsCount = count_objects(Conn), BatchesCount = round(math:ceil(ObjectsCount / ?BATCH_SIZE)), - lists:foreach( + ok = lists:foreach( fun(N) -> Objects0 = collect_objects(Conn, N * ?BATCH_SIZE, ?BATCH_SIZE), Objects1 = lists:map(fun form_search_vector/1, Objects0), insert_objects_into_buffer(Conn, Objects1) end, lists:seq(0, BatchesCount - 1) - ). + ), + ok = update_entities(Conn), + ok = drop_tmp_table(Conn). %% +create_tmp_table(Conn) -> + Query = """ + CREATE TABLE tmp_entity_search_vector ( + id TEXT NOT NULL, + version BIGINT NOT NULL REFERENCES version(version), + search_vector tsvector, + PRIMARY KEY (id, version) + ) + """, + case epgsql:squery(Conn, Query) of + {error, Error} -> erlang:throw(Error); + _ -> ok + end. + +update_entities(Conn) -> + Query = """ + UPDATE entity + SET search_vector = tmp.search_vector + FROM (SELECT id, version, search_vector FROM tmp_entity_search_vector) AS tmp + WHERE entity.id = tmp.id AND entity.version = tmp.version + """, + case epgsql:squery(Conn, Query) of + {error, Error} -> erlang:throw(Error); + _ -> ok + end. + +drop_tmp_table(Conn) -> + Query = """ + DROP TABLE tmp_entity_search_vector + """, + case epgsql:squery(Conn, Query) of + {error, Error} -> erlang:throw(Error); + _ -> ok + end. + count_objects(Conn) -> case epgsql:squery(Conn, "SELECT COUNT(*) FROM entity") of {ok, _Cols, [{Count}]} -> binary_to_integer(Count); diff --git a/migrations/1782722012-update-entities-with-search-vector-tmp-table.sql b/migrations/1782722012-update-entities-with-search-vector-tmp-table.sql deleted file mode 100644 index 1205760..0000000 --- a/migrations/1782722012-update-entities-with-search-vector-tmp-table.sql +++ /dev/null @@ -1,4 +0,0 @@ -UPDATE entity -SET search_vector = tmp.search_vector -FROM (SELECT id, version, search_vector FROM tmp_entity_search_vector) AS tmp -WHERE entity.id = tmp.id AND entity.version = tmp.version; diff --git a/migrations/1782722013-drop-search-vector-tmp-table.sql b/migrations/1782722013-drop-search-vector-tmp-table.sql deleted file mode 100644 index 2290f96..0000000 --- a/migrations/1782722013-drop-search-vector-tmp-table.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE tmp_entity_search_vector; From 591a2d979bc3bdd07d6a7a0dddea46ea5bab1076 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Mon, 6 Jul 2026 17:12:26 +0300 Subject: [PATCH 06/20] Refactors string to search vector formation --- apps/dmt/src/dmt_mapper.erl | 120 +++++++++++++++++- apps/dmt/src/dmt_repository.erl | 13 +- ...2011-fill-search-vector-fill-tmp-table.erl | 49 +++---- 3 files changed, 146 insertions(+), 36 deletions(-) diff --git a/apps/dmt/src/dmt_mapper.erl b/apps/dmt/src/dmt_mapper.erl index be8087e..c9ec8e7 100644 --- a/apps/dmt/src/dmt_mapper.erl +++ b/apps/dmt/src/dmt_mapper.erl @@ -10,9 +10,9 @@ -export([string_to_ref/1]). -export([object_to_string/1]). -export([string_to_object/1]). +-export([to_search_vector/2]). -export([to_string/1]). -export([from_string/1]). --export([extract_searchable_text_from_term/1]). -type row() :: tuple(). -type transform_fun() :: fun((map()) -> term()). @@ -111,6 +111,32 @@ object_to_string({_Type, _} = Data) -> string_to_object(Str) -> string_to_thrift_term_(Str, ?OBJECT_TYPE). +-spec to_search_vector(binary(), binary()) -> string(). +to_search_vector(RedundantRootKey, Str) -> + %% NOTE Turns string containing nested json into space-separated string of + %% words/lexems. + %% As example, this turns + %% { + %% "my-object": { + %% "ref": { + %% "id": "my-id" + %% }, + %% "data": { + %% "hello": "world", + %% "test": 42 + %% } + %% } + %% } + %% into + %% hello world test 42 + Json0 = + case jsx:decode(Str) of + #{RedundantRootKey := Inner} -> Inner; + Inner -> Inner + end, + Json1 = unwrap_redundant_json_nesting(Json0), + extract_searchable_text_from_term(Json1). + -spec thrift_term_to_string_(term(), dmt_thrift:thrift_type()) -> binary(). thrift_term_to_string_(Term, ThriftType) -> dmt_json:encode(dmt_json:term_to_json(Term, ThriftType)). @@ -192,6 +218,12 @@ extract_searchable_text_from_term(Term) -> TextList = extract_text(Term, []), join_text_list(TextList). +-spec unwrap_redundant_json_nesting(map()) -> map(). +unwrap_redundant_json_nesting(#{~"ref" := _, ~"data" := Data}) -> + unwrap_redundant_json_nesting(Data); +unwrap_redundant_json_nesting(Json0) -> + Json0. + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -229,4 +261,90 @@ stringify_object_test_() -> ?_assertEqual(Object, string_to_object(object_to_string(Object))) ]. +-spec to_search_vector_test_() -> _. +to_search_vector_test_() -> + %% NOTE Since JSON is decoded as map, ordering in output string is not + %% guaranteed to be the same w/ different Erlang versions. + [ + ?_assertEqual( + "hello world test 42", + to_search_vector( + ~"my-object", + ~""" + { + "my-object": { + "ref": { + "id": "my-id" + }, + "data": { + "hello": "world", + "test": 42 + } + } + } + """ + ) + ), + ?_assertEqual( + "my-object data hello world test 42 ref id my-id", + to_search_vector( + ~"other-root-key", + ~""" + { + "my-object": { + "ref": { + "id": "my-id" + }, + "data": { + "hello": "world", + "test": 42 + } + } + } + """ + ) + ), + ?_assertEqual( + "hello world test 42", + to_search_vector( + ~"my-object", + ~""" + { + "my-object": { + "hello": "world", + "test": 42 + } + } + """ + ) + ), + ?_assertEqual( + "", + to_search_vector( + ~"my-object", + ~""" + {} + """ + ) + ), + ?_assertEqual( + "string", + to_search_vector( + ~"my-object", + ~""" + "string" + """ + ) + ), + ?_assertEqual( + "", + to_search_vector( + ~"my-object", + ~""" + null + """ + ) + ) + ]. + -endif. diff --git a/apps/dmt/src/dmt_repository.erl b/apps/dmt/src/dmt_repository.erl index 3de4d62..db896b3 100644 --- a/apps/dmt/src/dmt_repository.erl +++ b/apps/dmt/src/dmt_repository.erl @@ -797,18 +797,9 @@ get_object_field({_, _, _, data, _}, Data, _Ref) -> Data. -spec object_to_serialized(domain_object()) -> {ok, JsonBinary :: binary(), SearchString :: string()}. -object_to_serialized(Data0) -> +object_to_serialized({Type, _} = Data0) -> Data1 = dmt_mapper:object_to_string(Data0), - %% NOTE Turns string containing nested json into space-separated string of - %% words/lexems. - %% As example, this turns - %% { - %% "hello": "world", - %% "test": 42 - %% } - %% into - %% hello world test 42 - SearchVector = dmt_mapper:extract_searchable_text_from_term(jsx:decode(Data1)), + SearchVector = dmt_mapper:to_search_vector(atom_to_binary(Type), Data1), {ok, Data1, SearchVector}. -spec update_object( diff --git a/migrations/1782722011-fill-search-vector-fill-tmp-table.erl b/migrations/1782722011-fill-search-vector-fill-tmp-table.erl index 9e93177..dea623f 100644 --- a/migrations/1782722011-fill-search-vector-fill-tmp-table.erl +++ b/migrations/1782722011-fill-search-vector-fill-tmp-table.erl @@ -2,6 +2,7 @@ -export([perform/2]). +-define(TMP_TABLE, "tmp_entity_search_vector"). -define(BATCH_SIZE, 500). perform(Conn, _MigrationOpts) -> @@ -22,35 +23,39 @@ perform(Conn, _MigrationOpts) -> %% create_tmp_table(Conn) -> - Query = """ - CREATE TABLE tmp_entity_search_vector ( - id TEXT NOT NULL, - version BIGINT NOT NULL REFERENCES version(version), - search_vector tsvector, - PRIMARY KEY (id, version) - ) - """, + Query = io_lib:format( + """ + CREATE TABLE ~s ( + id TEXT NOT NULL, + version BIGINT NOT NULL REFERENCES version(version), + search_vector tsvector, + PRIMARY KEY (id, version) + ) + """, + [?TMP_TABLE] + ), case epgsql:squery(Conn, Query) of {error, Error} -> erlang:throw(Error); _ -> ok end. update_entities(Conn) -> - Query = """ - UPDATE entity - SET search_vector = tmp.search_vector - FROM (SELECT id, version, search_vector FROM tmp_entity_search_vector) AS tmp - WHERE entity.id = tmp.id AND entity.version = tmp.version - """, + Query = io_lib:format( + """ + UPDATE entity + SET search_vector = tmp.search_vector + FROM (SELECT id, version, search_vector FROM ~s) AS tmp + WHERE entity.id = tmp.id AND entity.version = tmp.version + """, + [?TMP_TABLE] + ), case epgsql:squery(Conn, Query) of {error, Error} -> erlang:throw(Error); _ -> ok end. drop_tmp_table(Conn) -> - Query = """ - DROP TABLE tmp_entity_search_vector - """, + Query = io_lib:format("DROP TABLE ~s", [?TMP_TABLE]), case epgsql:squery(Conn, Query) of {error, Error} -> erlang:throw(Error); _ -> ok @@ -74,18 +79,14 @@ collect_objects(Conn, Offset, Limit) -> {error, Error} -> erlang:throw(Error) end. -form_search_vector({ID, Version, _Type, Data}) -> - SearchVector = dmt_mapper:extract_searchable_text_from_term(jsx:decode(Data)), +form_search_vector({ID, Version, Type, Data}) -> + SearchVector = dmt_mapper:to_search_vector(Type, Data), [ID, Version, SearchVector]. insert_objects_into_buffer(_Conn, []) -> ok; insert_objects_into_buffer(Conn, Objects) -> - QueryHead = """ - INSERT INTO tmp_entity_search_vector (id, version, search_vector) - VALUES - - """, + QueryHead = io_lib:format("INSERT INTO ~s (id, version, search_vector) VALUES ", [?TMP_TABLE]), Values = lists:join( $,, lists:map( From 1f71ab880fd02791e247c44962e14cc26bb4366e Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Mon, 6 Jul 2026 22:22:16 +0300 Subject: [PATCH 07/20] Adds `to_text_search_query` string pre-processing func for search --- apps/dmt/src/dmt_mapper.erl | 74 ++++++++++++++++--- apps/dmt/src/dmt_repository.erl | 2 +- ...2011-fill-search-vector-fill-tmp-table.erl | 2 +- 3 files changed, 65 insertions(+), 13 deletions(-) diff --git a/apps/dmt/src/dmt_mapper.erl b/apps/dmt/src/dmt_mapper.erl index c9ec8e7..7721e2d 100644 --- a/apps/dmt/src/dmt_mapper.erl +++ b/apps/dmt/src/dmt_mapper.erl @@ -10,7 +10,8 @@ -export([string_to_ref/1]). -export([object_to_string/1]). -export([string_to_object/1]). --export([to_search_vector/2]). +-export([to_text_search_vector/2]). +-export([to_text_search_query/1]). -export([to_string/1]). -export([from_string/1]). @@ -111,8 +112,8 @@ object_to_string({_Type, _} = Data) -> string_to_object(Str) -> string_to_thrift_term_(Str, ?OBJECT_TYPE). --spec to_search_vector(binary(), binary()) -> string(). -to_search_vector(RedundantRootKey, Str) -> +-spec to_text_search_vector(binary(), binary()) -> string(). +to_text_search_vector(RedundantRootKey, Str) -> %% NOTE Turns string containing nested json into space-separated string of %% words/lexems. %% As example, this turns @@ -137,6 +138,26 @@ to_search_vector(RedundantRootKey, Str) -> Json1 = unwrap_redundant_json_nesting(Json0), extract_searchable_text_from_term(Json1). +-spec to_text_search_query(binary()) -> binary(). +to_text_search_query(Query0) -> + {ok, NoSpecialRE} = re:compile(~"['\"&|!()\\\\]", [unicode, ucp]), + {ok, OnlyWordsRE} = re:compile(~"[^\\p{L}\\p{N}\\s\\-]", [unicode, ucp]), + F = fun(QueryPart0, Acc) -> + QueryPart1 = re:replace(QueryPart0, NoSpecialRE, ~"", [global]), + QueryPart2 = re:replace(QueryPart1, OnlyWordsRE, ~" ", [global]), + QueryPart3 = iolist_to_binary(QueryPart2), + case split_and_trim_into_keywords(QueryPart3) of + [] -> + Acc; + Keywords0 -> + Keywords1 = lists:map(fun(KW) -> <> end, Keywords0), + [binary:join(Keywords1, ~" & ") | Acc] + end + end, + Query1 = genlib_string:to_lower(Query0), + Query2 = lists:foldl(F, [], binary:split(Query1, <<$;>>, [global, trim_all])), + binary:join(lists:reverse(Query2), ~" | "). + -spec thrift_term_to_string_(term(), dmt_thrift:thrift_type()) -> binary(). thrift_term_to_string_(Term, ThriftType) -> dmt_json:encode(dmt_json:term_to_json(Term, ThriftType)). @@ -224,6 +245,22 @@ unwrap_redundant_json_nesting(#{~"ref" := _, ~"data" := Data}) -> unwrap_redundant_json_nesting(Json0) -> Json0. +-spec split_and_trim_into_keywords(binary()) -> [binary()]. +split_and_trim_into_keywords(Bin) -> + lists:reverse(split_and_trim_into_keywords(Bin, <<>>, [])). + +-spec split_and_trim_into_keywords(binary(), binary(), [binary()]) -> [binary()]. +split_and_trim_into_keywords(<<>>, <<>>, Acc) -> + Acc; +split_and_trim_into_keywords(<<>>, Keyword, Acc) -> + [Keyword | Acc]; +split_and_trim_into_keywords(<<$\s, Bin/binary>>, <<>>, Acc) -> + split_and_trim_into_keywords(Bin, <<>>, Acc); +split_and_trim_into_keywords(<<$\s, Bin/binary>>, Keyword, Acc) -> + split_and_trim_into_keywords(Bin, <<>>, [Keyword | Acc]); +split_and_trim_into_keywords(<>, Keyword, Acc) -> + split_and_trim_into_keywords(Bin, <>, Acc). + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -261,14 +298,14 @@ stringify_object_test_() -> ?_assertEqual(Object, string_to_object(object_to_string(Object))) ]. --spec to_search_vector_test_() -> _. -to_search_vector_test_() -> +-spec to_text_search_vector_test_() -> _. +to_text_search_vector_test_() -> %% NOTE Since JSON is decoded as map, ordering in output string is not %% guaranteed to be the same w/ different Erlang versions. [ ?_assertEqual( "hello world test 42", - to_search_vector( + to_text_search_vector( ~"my-object", ~""" { @@ -287,7 +324,7 @@ to_search_vector_test_() -> ), ?_assertEqual( "my-object data hello world test 42 ref id my-id", - to_search_vector( + to_text_search_vector( ~"other-root-key", ~""" { @@ -306,7 +343,7 @@ to_search_vector_test_() -> ), ?_assertEqual( "hello world test 42", - to_search_vector( + to_text_search_vector( ~"my-object", ~""" { @@ -320,7 +357,7 @@ to_search_vector_test_() -> ), ?_assertEqual( "", - to_search_vector( + to_text_search_vector( ~"my-object", ~""" {} @@ -329,7 +366,7 @@ to_search_vector_test_() -> ), ?_assertEqual( "string", - to_search_vector( + to_text_search_vector( ~"my-object", ~""" "string" @@ -338,7 +375,7 @@ to_search_vector_test_() -> ), ?_assertEqual( "", - to_search_vector( + to_text_search_vector( ~"my-object", ~""" null @@ -347,4 +384,19 @@ to_search_vector_test_() -> ) ]. +-spec to_text_search_query_test_() -> _. +to_text_search_query_test_() -> + [ + ?_assertEqual(~"test:*", to_text_search_query(~"test")), + ?_assertEqual(~"hello:* & world:*", to_text_search_query(~"hello world")), + ?_assertEqual( + ~"this:* & and:* & that:* | something:* & else:*", + to_text_search_query(~"this and that; something else") + ), + ?_assertEqual( + ~"weird-request:* & 42:* & special:* & symbols:*", + to_text_search_query(~"wEiRd-(rEQuesT)=!%{42[+\\/ spEcIAL@#sym&bols") + ) + ]. + -endif. diff --git a/apps/dmt/src/dmt_repository.erl b/apps/dmt/src/dmt_repository.erl index db896b3..59ab3f8 100644 --- a/apps/dmt/src/dmt_repository.erl +++ b/apps/dmt/src/dmt_repository.erl @@ -799,7 +799,7 @@ get_object_field({_, _, _, data, _}, Data, _Ref) -> -spec object_to_serialized(domain_object()) -> {ok, JsonBinary :: binary(), SearchString :: string()}. object_to_serialized({Type, _} = Data0) -> Data1 = dmt_mapper:object_to_string(Data0), - SearchVector = dmt_mapper:to_search_vector(atom_to_binary(Type), Data1), + SearchVector = dmt_mapper:to_text_search_vector(atom_to_binary(Type), Data1), {ok, Data1, SearchVector}. -spec update_object( diff --git a/migrations/1782722011-fill-search-vector-fill-tmp-table.erl b/migrations/1782722011-fill-search-vector-fill-tmp-table.erl index dea623f..9988e43 100644 --- a/migrations/1782722011-fill-search-vector-fill-tmp-table.erl +++ b/migrations/1782722011-fill-search-vector-fill-tmp-table.erl @@ -80,7 +80,7 @@ collect_objects(Conn, Offset, Limit) -> end. form_search_vector({ID, Version, Type, Data}) -> - SearchVector = dmt_mapper:to_search_vector(Type, Data), + SearchVector = dmt_mapper:to_text_search_vector(Type, Data), [ID, Version, SearchVector]. insert_objects_into_buffer(_Conn, []) -> From 0496bd65d19c6f3e5863c65ffe72f7ee47606360 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Mon, 6 Jul 2026 22:37:05 +0300 Subject: [PATCH 08/20] Adds notes to new func --- apps/dmt/src/dmt_mapper.erl | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/dmt/src/dmt_mapper.erl b/apps/dmt/src/dmt_mapper.erl index 7721e2d..184b661 100644 --- a/apps/dmt/src/dmt_mapper.erl +++ b/apps/dmt/src/dmt_mapper.erl @@ -138,6 +138,8 @@ to_text_search_vector(RedundantRootKey, Str) -> Json1 = unwrap_redundant_json_nesting(Json0), extract_searchable_text_from_term(Json1). +%% @doc Constructs text-search query for full-text search with lexeme's prefix +%% matching and OR binding via semicolon (";") character. -spec to_text_search_query(binary()) -> binary(). to_text_search_query(Query0) -> {ok, NoSpecialRE} = re:compile(~"['\"&|!()\\\\]", [unicode, ucp]), @@ -156,6 +158,9 @@ to_text_search_query(Query0) -> end, Query1 = genlib_string:to_lower(Query0), Query2 = lists:foldl(F, [], binary:split(Query1, <<$;>>, [global, trim_all])), + %% NOTE Parentheses can be used to enforce grouping of these operators. In + %% the absence of parentheses, ! (NOT) binds most tightly, <-> (FOLLOWED BY) + %% next most tightly, then & (AND), with | (OR) binding the least tightly. binary:join(lists:reverse(Query2), ~" | "). -spec thrift_term_to_string_(term(), dmt_thrift:thrift_type()) -> binary(). From 121ed2eebdb2fb768236ce0b7d2bf787db39bb0c Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Tue, 7 Jul 2026 15:04:23 +0300 Subject: [PATCH 09/20] Adds use of pre-processed search query w/ testcases --- apps/dmt/src/dmt_database.erl | 30 +-- apps/dmt/src/dmt_db_migration.erl | 2 +- apps/dmt/src/dmt_mapper.erl | 62 ++++-- apps/dmt/src/dmt_repository.erl | 2 +- .../test/dmt_search_objects_tests_SUITE.erl | 185 +++++++----------- ...2011-fill-search-vector-fill-tmp-table.erl | 2 +- 6 files changed, 136 insertions(+), 147 deletions(-) diff --git a/apps/dmt/src/dmt_database.erl b/apps/dmt/src/dmt_database.erl index 27ed3de..384776e 100644 --- a/apps/dmt/src/dmt_database.erl +++ b/apps/dmt/src/dmt_database.erl @@ -683,7 +683,7 @@ search_objects(Worker, <<"*">>, Version, Type, Limit, Offset) -> _ = logger:error("Error fetching search Params: ~p", [AllParams]), {ok, {[], undefined}} end; -search_objects(Worker, Query, Version, Type, Limit, Offset) -> +search_objects(Worker, Query0, Version, Type, Limit, Offset) -> % Use a pattern where the condition is always true when Type is NULL TypeValue = case Type of @@ -713,22 +713,23 @@ search_objects(Worker, Query, Version, Type, Limit, Offset) -> e.created_at FROM entity e INNER JOIN ActiveStatusAtRequestedTime las ON e.id = las.id - WHERE e.search_vector @@ plainto_tsquery('multilingual', $1) + WHERE e.search_vector @@ to_tsquery('multilingual', $1) AND e.version <= $2 AND ($3 = 'NULL' OR e.entity_type = $3) AND las.is_active = TRUE - ORDER BY e.id, e.version DESC, ts_rank(e.search_vector, plainto_tsquery('multilingual', $1)) DESC + ORDER BY e.id, e.version DESC, ts_rank(e.search_vector, to_tsquery('multilingual', $1)) DESC LIMIT $4 OFFSET $5 """, - AllParams = [Query, Version, TypeValue, Limit, Offset], + Query1 = dmt_mapper:to_text_search_query(Query0), + AllParams = [Query1, Version, TypeValue, Limit, Offset], % Execute the query case epg_pool:query(Worker, Request, AllParams) of {ok, Columns, Rows} -> Objects = dmt_mapper:to_marshalled_maps(Columns, Rows), NewOffset0 = Offset + Limit, - HasMoreResults = has_more_search_results(Worker, Query, Version, TypeValue, NewOffset0), + HasMoreResults = has_more_search_results(Worker, Query1, Version, TypeValue, NewOffset0), NewOffset1 = case HasMoreResults of true -> NewOffset0; @@ -795,7 +796,7 @@ has_more_search_results(Worker, Query, Version, TypeValue, Offset) -> ) SELECT 1 FROM entity e INNER JOIN ActiveStatusAtRequestedTime las ON e.id = las.id - WHERE e.search_vector @@ plainto_tsquery('multilingual', $1) + WHERE e.search_vector @@ to_tsquery('multilingual', $1) AND e.version <= $2 AND ($3 = 'NULL' OR e.entity_type = $3) AND las.is_active = TRUE @@ -1209,7 +1210,7 @@ get_multiple_related_graph_edges(Worker, ObjectRefStrings, Version, Depth, Inclu ) -> {ok, {[dmt_object:object()], [edge()]}} | {error, db_error()}. search_related_graph( - Worker, Query, SearchedType, Version, Depth, IncludeInbound, IncludeOutbound, ReturnedType + Worker, Query0, SearchedType, Version, Depth, IncludeInbound, IncludeOutbound, ReturnedType ) -> TypeValue = case SearchedType of @@ -1218,7 +1219,7 @@ search_related_graph( end, SearchQuery = - case Query of + case Query0 of ~"*" -> """ WITH LatestVersionAtRequestedTime AS ( @@ -1256,18 +1257,21 @@ search_related_graph( SELECT DISTINCT ON (e.id) e.id FROM entity e INNER JOIN ActiveStatusAtRequestedTime las ON e.id = las.id - WHERE e.search_vector @@ plainto_tsquery('multilingual', $1) + WHERE e.search_vector @@ to_tsquery('multilingual', $1) AND e.version <= $2 AND ($3 = 'NULL' OR e.entity_type = $3) AND las.is_active = TRUE - ORDER BY e.id, e.version DESC, ts_rank(e.search_vector, plainto_tsquery('multilingual', $1)) DESC + ORDER BY e.id, e.version DESC, ts_rank(e.search_vector, to_tsquery('multilingual', $1)) DESC """ end, SearchParams = - case Query of - ~"*" -> [Version, TypeValue]; - _ -> [Query, Version, TypeValue] + case Query0 of + ~"*" -> + [Version, TypeValue]; + _ -> + Query1 = dmt_mapper:to_text_search_query(Query0), + [Query1, Version, TypeValue] end, case epg_pool:query(Worker, SearchQuery, SearchParams) of diff --git a/apps/dmt/src/dmt_db_migration.erl b/apps/dmt/src/dmt_db_migration.erl index 066d1d3..f6c6bb1 100644 --- a/apps/dmt/src/dmt_db_migration.erl +++ b/apps/dmt/src/dmt_db_migration.erl @@ -32,7 +32,7 @@ run(MigrationsDir) -> -spec perform(map(), string()) -> ok | {error, term()}. perform(DbOpts, MigrationsDir) -> - case epg_migrator:perform(?REALM, DbOpts, [], MigrationsDir) of + case epg_migrator:perform(?REALM, DbOpts, [reraise], MigrationsDir) of {ok, Executed} -> _ = logger:info("Applied migrations: ~p", [Executed]), ok; diff --git a/apps/dmt/src/dmt_mapper.erl b/apps/dmt/src/dmt_mapper.erl index 184b661..95bee59 100644 --- a/apps/dmt/src/dmt_mapper.erl +++ b/apps/dmt/src/dmt_mapper.erl @@ -112,8 +112,8 @@ object_to_string({_Type, _} = Data) -> string_to_object(Str) -> string_to_thrift_term_(Str, ?OBJECT_TYPE). --spec to_text_search_vector(binary(), binary()) -> string(). -to_text_search_vector(RedundantRootKey, Str) -> +-spec to_text_search_vector(binary(), binary()) -> binary(). +to_text_search_vector(RedundantRootKey, Bin) -> %% NOTE Turns string containing nested json into space-separated string of %% words/lexems. %% As example, this turns @@ -131,12 +131,13 @@ to_text_search_vector(RedundantRootKey, Str) -> %% into %% hello world test 42 Json0 = - case jsx:decode(Str) of + case jsx:decode(Bin) of #{RedundantRootKey := Inner} -> Inner; Inner -> Inner end, Json1 = unwrap_redundant_json_nesting(Json0), - extract_searchable_text_from_term(Json1). + Str = extract_searchable_text_from_term(Json1), + to_unaccented_lowercase(unicode:characters_to_binary(Str)). %% @doc Constructs text-search query for full-text search with lexeme's prefix %% matching and OR binding via semicolon (";") character. @@ -144,7 +145,8 @@ to_text_search_vector(RedundantRootKey, Str) -> to_text_search_query(Query0) -> {ok, NoSpecialRE} = re:compile(~"['\"&|!()\\\\]", [unicode, ucp]), {ok, OnlyWordsRE} = re:compile(~"[^\\p{L}\\p{N}\\s\\-]", [unicode, ucp]), - F = fun(QueryPart0, Acc) -> + KeywordSuffixFun = fun(KW) -> <> end, + QueryProcessorFun = fun(QueryPart0, Acc) -> QueryPart1 = re:replace(QueryPart0, NoSpecialRE, ~"", [global]), QueryPart2 = re:replace(QueryPart1, OnlyWordsRE, ~" ", [global]), QueryPart3 = iolist_to_binary(QueryPart2), @@ -152,12 +154,13 @@ to_text_search_query(Query0) -> [] -> Acc; Keywords0 -> - Keywords1 = lists:map(fun(KW) -> <> end, Keywords0), + Keywords1 = lists:map(KeywordSuffixFun, Keywords0), [binary:join(Keywords1, ~" & ") | Acc] end end, - Query1 = genlib_string:to_lower(Query0), - Query2 = lists:foldl(F, [], binary:split(Query1, <<$;>>, [global, trim_all])), + %% Query1 = genlib_string:to_lower(Query0), + Query1 = to_unaccented_lowercase(Query0), + Query2 = lists:foldl(QueryProcessorFun, [], binary:split(Query1, <<$;>>, [global, trim_all])), %% NOTE Parentheses can be used to enforce grouping of these operators. In %% the absence of parentheses, ! (NOT) binds most tightly, <-> (FOLLOWED BY) %% next most tightly, then & (AND), with | (OR) binding the least tightly. @@ -266,6 +269,19 @@ split_and_trim_into_keywords(<<$\s, Bin/binary>>, Keyword, Acc) -> split_and_trim_into_keywords(<>, Keyword, Acc) -> split_and_trim_into_keywords(Bin, <>, Acc). +-spec to_unaccented_lowercase(binary()) -> binary(). +to_unaccented_lowercase(Str0) -> + {ok, Re} = re:compile("\\p{Mn}", [unicode]), + Str1 = throw_if_bad_binary(unicode:characters_to_nfd_binary(Str0)), + Str2 = re:replace(Str1, Re, "", [global]), + Str3 = throw_if_bad_binary(unicode:characters_to_nfc_binary(Str2)), + throw_if_bad_binary(unicode:characters_to_binary(string:lowercase(Str3))). + +-spec throw_if_bad_binary(binary() | {error, _, _}) -> binary() | no_return(). +throw_if_bad_binary({error, _, _}) -> erlang:throw(bad_binary); +throw_if_bad_binary({incomplete, _, _}) -> erlang:throw(bad_binary); +throw_if_bad_binary(V) -> V. + -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). @@ -309,7 +325,25 @@ to_text_search_vector_test_() -> %% guaranteed to be the same w/ different Erlang versions. [ ?_assertEqual( - "hello world test 42", + ~"key значение с кириллицеи и акцентом е", + to_text_search_vector( + ~"my-object", + ~""" + { + "my-object": { + "ref": { + "id": "my-id" + }, + "data": { + "KEY": "Значение с кириллицей и акцентом Ё" + } + } + } + """ + ) + ), + ?_assertEqual( + ~"hello world test 42", to_text_search_vector( ~"my-object", ~""" @@ -328,7 +362,7 @@ to_text_search_vector_test_() -> ) ), ?_assertEqual( - "my-object data hello world test 42 ref id my-id", + ~"my-object data hello world test 42 ref id my-id", to_text_search_vector( ~"other-root-key", ~""" @@ -347,7 +381,7 @@ to_text_search_vector_test_() -> ) ), ?_assertEqual( - "hello world test 42", + ~"hello world test 42", to_text_search_vector( ~"my-object", ~""" @@ -361,7 +395,7 @@ to_text_search_vector_test_() -> ) ), ?_assertEqual( - "", + ~"", to_text_search_vector( ~"my-object", ~""" @@ -370,7 +404,7 @@ to_text_search_vector_test_() -> ) ), ?_assertEqual( - "string", + ~"string", to_text_search_vector( ~"my-object", ~""" @@ -379,7 +413,7 @@ to_text_search_vector_test_() -> ) ), ?_assertEqual( - "", + ~"", to_text_search_vector( ~"my-object", ~""" diff --git a/apps/dmt/src/dmt_repository.erl b/apps/dmt/src/dmt_repository.erl index 59ab3f8..5688409 100644 --- a/apps/dmt/src/dmt_repository.erl +++ b/apps/dmt/src/dmt_repository.erl @@ -796,7 +796,7 @@ get_object_field({_, _, _, ref, _}, _Data, {_Type, Ref}) -> get_object_field({_, _, _, data, _}, Data, _Ref) -> Data. --spec object_to_serialized(domain_object()) -> {ok, JsonBinary :: binary(), SearchString :: string()}. +-spec object_to_serialized(domain_object()) -> {ok, JsonBinary :: binary(), SearchString :: binary()}. object_to_serialized({Type, _} = Data0) -> Data1 = dmt_mapper:object_to_string(Data0), SearchVector = dmt_mapper:to_text_search_vector(atom_to_binary(Type), Data1), diff --git a/apps/dmt/test/dmt_search_objects_tests_SUITE.erl b/apps/dmt/test/dmt_search_objects_tests_SUITE.erl index d0a50a4..0306c77 100644 --- a/apps/dmt/test/dmt_search_objects_tests_SUITE.erl +++ b/apps/dmt/test/dmt_search_objects_tests_SUITE.erl @@ -34,10 +34,10 @@ search_full_objects_basic_test/1, search_full_objects_with_filter_test/1, search_full_objects_pagination_test/1, - search_objects_without_name_desc_test/1, search_deleted_objects_test/1, checkout_deleted_version_test/1, - search_updated_object_deduplication_test/1 + search_updated_object_deduplication_test/1, + search_prefix_match_test/1 ]). %% Initialize per suite @@ -68,14 +68,14 @@ groups() -> search_with_version_filter_test, search_multiple_terms_test, search_no_results_test, - search_objects_without_name_desc_test, search_invalid_query_test, search_full_objects_basic_test, search_full_objects_with_filter_test, search_full_objects_pagination_test, search_deleted_objects_test, checkout_deleted_version_test, - search_updated_object_deduplication_test + search_updated_object_deduplication_test, + search_prefix_match_test ]} ]. @@ -1028,118 +1028,6 @@ search_full_objects_pagination_test(Config) -> length(AllRefs), length(UniqueRefs), "Should not have any duplicate results across pages" ). -% Test searching for objects without name or description fields -search_objects_without_name_desc_test(Config) -> - Client = dmt_ct_helper:cfg(client, Config), - - % Create author - Email = <<"search_objects_without_name_desc_test@test">>, - AuthorID = create_author(Email, Client), - - % Ensure the 'dummy' entity type exists in the database, it could be deleted by previous tests - {ok, _} = epg_pool:query( - default_pool, - """ - INSERT INTO entity_type (name, has_sequence) - VALUES ($1, FALSE) - ON CONFLICT (name) DO NOTHING; - """, - [dummy] - ), - - % Create a Dummy object which doesn't have name or description fields - Revision = 0, - DummyRef = #domain_DummyRef{id = <<"search_objects_without_name_desc_test">>}, - Operations = [ - {insert, #domain_conf_v2_InsertOp{ - force_ref = {dummy, DummyRef}, - object = {dummy, #domain_Dummy{}} - }} - ], - - {ok, #domain_conf_v2_CommitResponse{ - version = Version, - new_objects = NewObjects - }} = dmt_client:commit(Revision, Operations, AuthorID, Client), - - % Extract the created dummy reference - [{dummy, #domain_DummyObject{ref = DummyRef}}] = ordsets:to_list(NewObjects), - - % Search for dummy objects (should find by type since there's no name/description) - Request = #domain_conf_v2_SearchRequestParams{ - % This should match the type name - query = <<"dummy">>, - version = Version, - limit = 10 - }, - - {ok, #domain_conf_v2_SearchResponse{ - result = Results, - total_count = Count - }} = dmt_client:search_objects(Request, Client), - - % We should still find the object even though it doesn't have name/description - ?assertEqual(1, Count, "Should find the dummy object"), - - % Verify the result structure - ?assertMatch( - [ - #domain_conf_v2_LimitedVersionedObject{ - ref = {dummy, DummyRef}, - info = #domain_conf_v2_VersionedObjectInfo{ - version = Version - }, - name = undefined, - description = undefined - } - ], - Results, - "Should return the dummy object with undefined name and description" - ), - - % Also search with explicit type filter - TypedRequest = #domain_conf_v2_SearchRequestParams{ - % Empty query but with type filter - query = <<"*">>, - version = Version, - limit = 10, - type = dummy - }, - - {ok, #domain_conf_v2_SearchResponse{ - result = TypedResults, - total_count = TypedCount - }} = dmt_client:search_objects(TypedRequest, Client), - - % Should find by type even with empty query - ?assertEqual(1, TypedCount, "Should find the dummy object by type filter"), - ?assertEqual(Results, TypedResults, "Same result should be returned for type-based search"), - - % Also test with full objects search - {ok, #domain_conf_v2_SearchFullResponse{ - result = FullResults, - total_count = FullCount - }} = dmt_client:search_full_objects(Request, Client), - - % Verify full object search works too - ?assertEqual(1, FullCount, "Should find the dummy object in full search"), - ?assertMatch( - [ - #domain_conf_v2_VersionedObject{ - info = #domain_conf_v2_VersionedObjectInfo{ - version = Version - }, - object = - {dummy, #domain_DummyObject{ - ref = DummyRef, - data = #domain_Dummy{} - }} - } - ], - FullResults, - "Should return the full dummy object" - ). - % Test searching for deleted objects search_deleted_objects_test(Config) -> Client = dmt_ct_helper:cfg(client, Config), @@ -1588,7 +1476,59 @@ search_updated_object_deduplication_test(Config) -> ?assertEqual(UpdatedName2, FullName, "Full object should have the latest name"), ?assertEqual(UpdatedDesc2, FullDesc, "Full object should have the latest description"). -%% Helper function +-define(assertFound(Count, Query, Client), + ?assertMatch( + #domain_conf_v2_SearchResponse{total_count = Count}, + search(Query, 10, Client), + unicode:characters_to_binary( + io_lib:format("Failed to assert that search with '~s' finds ~c results", [Query, Count]) + ) + ) +). + +search_prefix_match_test(Config) -> + Client = dmt_ct_helper:cfg(client, Config), + ok = insert( + [ + {category, #domain_Category{ + name = ~"Main regularly reward", + description = ~""" + Hard hopéful hope widé kindly angle truth dock quiet two clearly + all. Us real airport learn doorway problem most catch because + first local sing always simple drink. Join decision push honor + hot speed, doctor similar valley read think press builder + Wédnésday watch try airport available certainly wire. + """ + }}, + {category, #domain_Category{ + name = ~"Йнцидент не исчёрпан: кровь стынЁт в жЙлах", + description = ~""" + Ясность нашей позиции очевидна: постоянный количественный рост и + сфера нашей активности прекрасно подходит для реализации + дальнейших направлений развития. Господа, глубокий уровень + погружения говорит о возможностях поставленных обществом задач. + """ + }} + ], + create_author(~"search_prefix_match_test@test", Client), + Client + ), + ?assertFound(1, ~"main reward", Client), + ?assertFound(0, ~"ololo not found", Client), + ?assertFound(1, ~"hard hop", Client), + ?assertFound(1, ~"wednesd", Client), + ?assertFound(1, ~"local sing always simple drink", Client), + ?assertFound(1, ~"инцидент исчерпан", Client), + ?assertFound(1, ~"стын", Client), + ?assertFound(1, ~"возможно обще", Client), + ?assertFound(0, ~"совершенно другой текст", Client), + ?assertFound(1, ~"совершенно другой текст;реал", Client), + ?assertFound(1, ~"количество сфера", Client), + ?assertFound(2, ~"количество сфера; air wir", Client), + ok. + +%% Helper functions + create_author(Email, Client) -> AuthorParams = #domain_conf_v2_AuthorParams{ email = Email, @@ -1596,3 +1536,14 @@ create_author(Email, Client) -> }, {ok, #domain_conf_v2_Author{id = AuthorID}} = dmt_client:create_author(AuthorParams, Client), AuthorID. + +insert(Objects, AuthorID, Client) -> + InsertOperations = [{insert, #domain_conf_v2_InsertOp{object = O}} || O <- Objects], + {ok, #domain_conf_v2_CommitResponse{version = _, new_objects = _}} = + dmt_client:commit(0, InsertOperations, AuthorID, Client), + ok. + +search(Query, Limit, Client) -> + {ok, #domain_conf_v2_SearchResponse{} = Result} = + dmt_client:search_objects(#domain_conf_v2_SearchRequestParams{query = Query, limit = Limit}, Client), + Result. diff --git a/migrations/1782722011-fill-search-vector-fill-tmp-table.erl b/migrations/1782722011-fill-search-vector-fill-tmp-table.erl index 9988e43..9f9eed1 100644 --- a/migrations/1782722011-fill-search-vector-fill-tmp-table.erl +++ b/migrations/1782722011-fill-search-vector-fill-tmp-table.erl @@ -90,7 +90,7 @@ insert_objects_into_buffer(Conn, Objects) -> Values = lists:join( $,, lists:map( - fun({I, [ID, Version, SearchVector]}) -> + fun({I, [_ID, _Version, _SearchVector]}) -> PH = [ph(I, 1), ph(I, 2), ["to_tsvector('multilingual',", ph(I, 3), ")"]], [$(, lists:join($,, PH), $)] end, From 9c3204ffe60f069a3e7b818f20fdbae401ad4358 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Tue, 7 Jul 2026 16:43:45 +0300 Subject: [PATCH 10/20] Adds host's uid and gid as dev user inside `Dockerfile.dev` image --- .github/workflows/erlang-checks.yml | 2 +- Dockerfile.dev | 8 +++++++- Makefile | 6 ++++-- apps/dmt/src/dmt_mapper.erl | 1 - compose.yaml | 2 ++ 5 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.github/workflows/erlang-checks.yml b/.github/workflows/erlang-checks.yml index 2e6caee..2fb12cf 100644 --- a/.github/workflows/erlang-checks.yml +++ b/.github/workflows/erlang-checks.yml @@ -30,7 +30,7 @@ jobs: run: name: Run checks needs: setup - uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@v1 + uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@527977799bb5b747c0110d4b47c9b6154f5ba988 with: otp-version: ${{ needs.setup.outputs.otp-version }} rebar-version: ${{ needs.setup.outputs.rebar-version }} diff --git a/Dockerfile.dev b/Dockerfile.dev index 4c0b0f0..ef6b151 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -11,9 +11,15 @@ RUN wget -q -O- "https://github.com/valitydev/thrift/releases/download/${THRIFT_ RUN apt-get update && apt-get install -y cmake +# Setup user for matching permissions in mounted volumes +ARG USER_UID +ARG USER_GID +RUN useradd -ms /bin/bash dev +USER dev + # Set env ENV CHARSET=UTF-8 ENV LANG=C.UTF-8 # Set runtime -CMD ["/bin/bash"] \ No newline at end of file +CMD ["/bin/bash"] diff --git a/Makefile b/Makefile index d16562f..a81aee9 100644 --- a/Makefile +++ b/Makefile @@ -13,10 +13,12 @@ DOTENV := $(shell grep -v '^\#' .env) # Development images DEV_IMAGE_TAG = $(TEST_CONTAINER_NAME)-dev DEV_IMAGE_ID = $(file < .image.dev) +USER_UID=$(shell id -u) +USER_GID=$(shell id -g) DOCKER ?= docker DOCKERCOMPOSE ?= docker compose -DOCKERCOMPOSE_W_ENV = DEV_IMAGE_TAG=$(DEV_IMAGE_TAG) $(DOCKERCOMPOSE) -f compose.yaml -f compose.tracing.yaml +DOCKERCOMPOSE_W_ENV = USER_UID=$(USER_UID) USER_GID=$(USER_GID) DEV_IMAGE_TAG=$(DEV_IMAGE_TAG) $(DOCKERCOMPOSE) -f compose.yaml -f compose.tracing.yaml REBAR ?= rebar3 TEST_CONTAINER_NAME ?= testrunner @@ -51,7 +53,7 @@ wc-%: dev-image $(DOCKER_RUN) $(DEV_IMAGE_TAG) make $* wdeps-shell: dev-image - $(DOCKERCOMPOSE_RUN) $(TEST_CONTAINER_NAME) su; \ + $(DOCKERCOMPOSE_RUN) $(TEST_CONTAINER_NAME) bash; \ $(DOCKERCOMPOSE_W_ENV) down wdeps-%: dev-image diff --git a/apps/dmt/src/dmt_mapper.erl b/apps/dmt/src/dmt_mapper.erl index 95bee59..07fc389 100644 --- a/apps/dmt/src/dmt_mapper.erl +++ b/apps/dmt/src/dmt_mapper.erl @@ -158,7 +158,6 @@ to_text_search_query(Query0) -> [binary:join(Keywords1, ~" & ") | Acc] end end, - %% Query1 = genlib_string:to_lower(Query0), Query1 = to_unaccented_lowercase(Query0), Query2 = lists:foldl(QueryProcessorFun, [], binary:split(Query1, <<$;>>, [global, trim_all])), %% NOTE Parentheses can be used to enforce grouping of these operators. In diff --git a/compose.yaml b/compose.yaml index e573ecd..3128148 100644 --- a/compose.yaml +++ b/compose.yaml @@ -14,6 +14,8 @@ services: args: OTP_VERSION: $OTP_VERSION THRIFT_VERSION: $THRIFT_VERSION + USER_UID: $USER_UID + USER_GID: $USER_GID volumes: - .:$PWD hostname: dmt.default From 61f13c47443fdfa32ef6173330856a4ff3b6bc48 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Tue, 7 Jul 2026 16:53:33 +0300 Subject: [PATCH 11/20] Bumps `cache-version` --- .github/workflows/erlang-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/erlang-checks.yml b/.github/workflows/erlang-checks.yml index 2fb12cf..fe5bd6a 100644 --- a/.github/workflows/erlang-checks.yml +++ b/.github/workflows/erlang-checks.yml @@ -38,4 +38,4 @@ jobs: thrift-version: ${{ needs.setup.outputs.thrift-version }} run-ct-with-compose: true upload-coverage: false - cache-version: v4 + cache-version: v5 From 5b98340ae4166cefd0177ce52f7746bbfb6daa04 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Tue, 7 Jul 2026 18:58:04 +0300 Subject: [PATCH 12/20] Debug permissions WF --- .github/workflows/erlang-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/erlang-checks.yml b/.github/workflows/erlang-checks.yml index fe5bd6a..8ae1690 100644 --- a/.github/workflows/erlang-checks.yml +++ b/.github/workflows/erlang-checks.yml @@ -30,7 +30,7 @@ jobs: run: name: Run checks needs: setup - uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@527977799bb5b747c0110d4b47c9b6154f5ba988 + uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@ddb84107d75c8286694f2f2b61fcae6efe0fcd88 with: otp-version: ${{ needs.setup.outputs.otp-version }} rebar-version: ${{ needs.setup.outputs.rebar-version }} From a7d8825bf1d0c405d1f7810882625692e9401047 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Tue, 7 Jul 2026 19:09:51 +0300 Subject: [PATCH 13/20] Bumps ci wf action --- .github/workflows/erlang-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/erlang-checks.yml b/.github/workflows/erlang-checks.yml index 8ae1690..c4631ee 100644 --- a/.github/workflows/erlang-checks.yml +++ b/.github/workflows/erlang-checks.yml @@ -30,7 +30,7 @@ jobs: run: name: Run checks needs: setup - uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@ddb84107d75c8286694f2f2b61fcae6efe0fcd88 + uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@7c5c53ab838da69bc1ac9e0e35b6d78858694cdc with: otp-version: ${{ needs.setup.outputs.otp-version }} rebar-version: ${{ needs.setup.outputs.rebar-version }} From 5f092236fe587d846469a6545aaeaa2c1eefbe4a Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Tue, 7 Jul 2026 19:16:59 +0300 Subject: [PATCH 14/20] Bumps ci wf action --- .github/workflows/erlang-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/erlang-checks.yml b/.github/workflows/erlang-checks.yml index c4631ee..b97d54d 100644 --- a/.github/workflows/erlang-checks.yml +++ b/.github/workflows/erlang-checks.yml @@ -30,7 +30,7 @@ jobs: run: name: Run checks needs: setup - uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@7c5c53ab838da69bc1ac9e0e35b6d78858694cdc + uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@694c874bc8317d0c8fcd9f7e4def77cdfbad71c3 with: otp-version: ${{ needs.setup.outputs.otp-version }} rebar-version: ${{ needs.setup.outputs.rebar-version }} From 43e19fdf37645321ad394e8e5bce449b98c6efad Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Tue, 7 Jul 2026 19:25:08 +0300 Subject: [PATCH 15/20] Adds user uid/gid to compose --- compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/compose.yaml b/compose.yaml index 3128148..d88fa1c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -16,6 +16,7 @@ services: THRIFT_VERSION: $THRIFT_VERSION USER_UID: $USER_UID USER_GID: $USER_GID + user: "${USER_UID}:${USER_GID}" volumes: - .:$PWD hostname: dmt.default From da87ff672c8aee7db87078aca6f06db5658ea33e Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Tue, 7 Jul 2026 20:11:11 +0300 Subject: [PATCH 16/20] Bumps ci wf action --- .github/workflows/erlang-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/erlang-checks.yml b/.github/workflows/erlang-checks.yml index b97d54d..80769ef 100644 --- a/.github/workflows/erlang-checks.yml +++ b/.github/workflows/erlang-checks.yml @@ -30,7 +30,7 @@ jobs: run: name: Run checks needs: setup - uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@694c874bc8317d0c8fcd9f7e4def77cdfbad71c3 + uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@97ea610ac4200a5d865615ce6776686ad30b296d with: otp-version: ${{ needs.setup.outputs.otp-version }} rebar-version: ${{ needs.setup.outputs.rebar-version }} From cc7c4dcbf031d7227da6d25b20176039a81af1c9 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Tue, 7 Jul 2026 20:27:04 +0300 Subject: [PATCH 17/20] Bumps ci wf action --- .github/workflows/erlang-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/erlang-checks.yml b/.github/workflows/erlang-checks.yml index 80769ef..5c81868 100644 --- a/.github/workflows/erlang-checks.yml +++ b/.github/workflows/erlang-checks.yml @@ -30,7 +30,7 @@ jobs: run: name: Run checks needs: setup - uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@97ea610ac4200a5d865615ce6776686ad30b296d + uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@4855ca2f829fd8aa0b3189b8f706f2fb5f428943 with: otp-version: ${{ needs.setup.outputs.otp-version }} rebar-version: ${{ needs.setup.outputs.rebar-version }} From e23bf23f24a331d5c18a6175a2fc4e43e6977654 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Thu, 9 Jul 2026 11:38:45 +0300 Subject: [PATCH 18/20] Bumps ci wf --- .github/workflows/erlang-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/erlang-checks.yml b/.github/workflows/erlang-checks.yml index 5c81868..895c007 100644 --- a/.github/workflows/erlang-checks.yml +++ b/.github/workflows/erlang-checks.yml @@ -30,7 +30,7 @@ jobs: run: name: Run checks needs: setup - uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@4855ca2f829fd8aa0b3189b8f706f2fb5f428943 + uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@5b5b09ecfccfdf758d4daecdfa0c31cc9e4ed8bd with: otp-version: ${{ needs.setup.outputs.otp-version }} rebar-version: ${{ needs.setup.outputs.rebar-version }} From f0ea80413ca38049e8dd5db39648d02e1b31f3d8 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Tue, 14 Jul 2026 14:14:16 +0300 Subject: [PATCH 19/20] Makes use of uid/gid args explicit and passes them thru w/ defaults --- Dockerfile.dev | 7 ++++--- Makefile | 4 ++-- compose.yaml | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/Dockerfile.dev b/Dockerfile.dev index ef6b151..2ead6be 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -12,9 +12,10 @@ RUN wget -q -O- "https://github.com/valitydev/thrift/releases/download/${THRIFT_ RUN apt-get update && apt-get install -y cmake # Setup user for matching permissions in mounted volumes -ARG USER_UID -ARG USER_GID -RUN useradd -ms /bin/bash dev +ARG USER_UID=1000 +ARG USER_GID=1000 +RUN groupadd -g ${USER_GID} dev && \ + useradd -m -s /bin/bash -u ${USER_UID} -g ${USER_GID} dev USER dev # Set env diff --git a/Makefile b/Makefile index a81aee9..1bb4348 100644 --- a/Makefile +++ b/Makefile @@ -13,8 +13,8 @@ DOTENV := $(shell grep -v '^\#' .env) # Development images DEV_IMAGE_TAG = $(TEST_CONTAINER_NAME)-dev DEV_IMAGE_ID = $(file < .image.dev) -USER_UID=$(shell id -u) -USER_GID=$(shell id -g) +USER_UID:=$(shell id -u) +USER_GID:=$(shell id -g) DOCKER ?= docker DOCKERCOMPOSE ?= docker compose diff --git a/compose.yaml b/compose.yaml index d88fa1c..505a12d 100644 --- a/compose.yaml +++ b/compose.yaml @@ -16,7 +16,7 @@ services: THRIFT_VERSION: $THRIFT_VERSION USER_UID: $USER_UID USER_GID: $USER_GID - user: "${USER_UID}:${USER_GID}" + user: "${USER_UID:-1000}:${USER_GID:-1000}" volumes: - .:$PWD hostname: dmt.default From 8c5da58c635cf0d89e4c2cebe59e056f5764a126 Mon Sep 17 00:00:00 2001 From: Aleksey Kashapov Date: Tue, 14 Jul 2026 18:01:51 +0300 Subject: [PATCH 20/20] Bumps CI action to v2 --- .github/workflows/erlang-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/erlang-checks.yml b/.github/workflows/erlang-checks.yml index 895c007..283f4d9 100644 --- a/.github/workflows/erlang-checks.yml +++ b/.github/workflows/erlang-checks.yml @@ -30,7 +30,7 @@ jobs: run: name: Run checks needs: setup - uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@5b5b09ecfccfdf758d4daecdfa0c31cc9e4ed8bd + uses: valitydev/erlang-workflows/.github/workflows/erlang-parallel-build.yml@v2 with: otp-version: ${{ needs.setup.outputs.otp-version }} rebar-version: ${{ needs.setup.outputs.rebar-version }}