This commit is contained in:
2026-06-02 10:32:53 -07:00
parent 55a56753c1
commit ef69016294
2 changed files with 10 additions and 5 deletions
+5 -5
View File
@@ -17,7 +17,7 @@ main() ->
% directory containing the tests for the tokenizer
ct_dir() ->
zx_daemon:get_home() ++ "/ct/".
zx_daemon:get_home() ++ "/ct".
agreement_tests_dir() ->
ct_dir() ++ "/tokenizers_agree".
@@ -144,14 +144,14 @@ div_test_() ->
tokens_match(FileName, FilePath) ->
%?debugFmt("tokens_match(~p, _)", [FileName]),
% extracting data to be tested
SoTokens = sfp:so_tokens(FilePath),
SfTokens = sfp:gsc_so_tokens(FilePath),
SoTokens = gsc_cli:so_tokens(FilePath),
SfTokens = gsc_cli:gso_tokens(FilePath),
{FileName ++ ": tokenizers_agree",
fun() ->
case {SoTokens, SfTokens} of
{{ok, So}, {ok, Sf}} -> ?assertEqual(So, Sf);
{{error, _}, {error, _}} -> ok;
{{ok, _}, {error, _}} -> error("so_scan succeeded and gsc_so_scan failed");
{{error, _}, {ok, _}} -> error("so_scan failed and gsc_so_scan succeded")
{{ok, _}, {error, _}} -> error("so_scan succeeded and gso_scan failed");
{{error, _}, {ok, _}} -> error("so_scan failed and gso_scan succeded")
end
end}.
+5
View File
@@ -8,6 +8,11 @@
-copyright("Peter Harpending <peterharpending@qpq.swiss>").
-license("GPL-3.0-only").
-export([
tokens/1,
so_tokens/1,
gso_tokens/1
]).
-export([start/1]).
-include("$gsc_include/gsc.hrl").