Test

.gitattributes

@@ -0,0 +1,4 @@
+scriptshifter/hooks/seq2seq/**/*.csv filter=lfs diff=lfs merge=lfs -text
+scriptshifter/hooks/seq2seq/**/*.json filter=lfs diff=lfs merge=lfs -text
+scriptshifter/hooks/seq2seq/**/*.pth filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -142,3 +142,5 @@ scriptshifter/data/*.db
 VERSION
 .~lock.*
 /data
+# Tokenizer data can be rebuilt deterministically within a few seconds.
+/scriptshifter/hooks/seq2seq/data/tokenizer/**

doc/gen_script_table_doc.py

@@ -2,11 +2,20 @@
 
 import sys
 from os import path
+from yaml import load
+try:
+    from yaml import CLoader as Loader
+except ImportError:
+    from yaml import Loader
+
 
 DOCROOT = path.dirname(__file__)
 
 sys.path.append(path.dirname(DOCROOT))
-from scriptshifter.tables import list_tables
+from scriptshifter.tables import TABLE_DIR, list_tables
+
+REL_TABLE_DIR = "../scriptshifter/tables/data"
+
 
 with open(path.join(DOCROOT, "supported_scripts_template.md"), "r") as fh:
     tpl_data = fh.read()
@@ -18,13 +27,19 @@
         if "alias_of" in data:
             fh.write(
                 f"| `{name}` | {data['label']} | "
-                f"- | - | [`{data['alias_of']}`]"
-                f"(../scriptshifter/tables/data/{data['alias_of']}.yml) |\n"
+                f"[`{data['alias_of']}`]({REL_TABLE_DIR}/{data['alias_of']}.yml)"
+                f" | - | - | - | -\n"
             )
         else:
+            fname = path.join(TABLE_DIR, name + ".yml")
+            with open(fname, "r") as md_fh:
+                md = load(md_fh, Loader=Loader)
+
             fh.write(
-                f"| [`{name}`](../scriptshifter/tables/data/{name}.yml) | "
-                f"{data['label']} | "
+                f"| [`{name}`]({REL_TABLE_DIR}/{name}.yml) | "
+                f"{data['label']} | - | "
                 f"{'Y' if data.get('has_r2s', False) else 'N'} | "
-                f"{'Y' if data.get('has_s2r', False) else 'N'} | - |\n"
+                f"{'Y' if data.get('has_s2r', False) else 'N'} | "
+                f"{md['general'].get('version', '-')} | "
+                f"{md['general'].get('date', '-')} \n"
             )

doc/supported_scripts_template.md

@@ -5,5 +5,5 @@ Below are the supported scripts, and supported directionality of those scripts,
 "Alias of" means that the script table is the same as the table indicated, only
 under a different name.
 
-| Mapping file |  Script Name  |  Roman-to-script  |  Script-to-roman | Alias of |
-| -------- | ------- | ------- | ------- | ------- |
+| Mapping file |  Script Name  |  Alias of | Roman-to-script  |  Script-to-roman | Version | Last updated
+| -------- | ------- | ------- | ------- | ------- | -------  | -------  |

requirements.txt

@@ -4,14 +4,14 @@ camel-tools>=1.6
 #esupar>=1.9,<2  # For Thai. Needs training. Breaks whole container.
 flask>=2.3,<3
 flask-cors>=4.0,<5
-#funcy>=1.15,<2
+funcy>=1.15,<2
 piraye>=1.1,<2
-#pymarc>=4.0,<5
+pymarc>=4.0,<5
 python-dotenv>=1.0,<2
 pyyaml>=6.0,<7
-#regex>=2023.8.8
+regex>=2023.8.8
 shekar>=1.6,<2
-#repackage>=0.7.3
+repackage>=0.7.3
 setuptools<81 # For yiddish
 tokenizers>=0.22
 torch>=2.12,<3