From a7356162015b663cde25c3609ad3bed4ec69bbd8 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Fri, 28 Aug 2026 23:57:27 -0400 Subject: [PATCH 1/9] docs: add NOTICE for redistributed data source terms - carry the two Government of Canada notices the open licence requires verbatim - record FAA public-domain provenance and unassessed third-party republished assets - LICENSE covers code only; release data carries its own conditions Generated-by: Claude Opus 5 --- NOTICE | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 NOTICE diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..22726d7 --- /dev/null +++ b/NOTICE @@ -0,0 +1,57 @@ +OpenAirframes — Data Source Notices +=================================== + +The LICENSE file covers the *code* in this repository. It does not cover the +*data* published in releases. Several upstream registries permit redistribution +only on condition that specific notices travel with the data. Those conditions +are reproduced below and must ship alongside any redistributed release asset. + +Removing or altering a notice in this file removes the permission that makes the +corresponding asset redistributable. + + +Transport Canada — Canadian Civil Aircraft Register (CCARCS) +------------------------------------------------------------ +Asset: openairframes_tc_*.csv +Source: https://wwwapps.tc.gc.ca/saf-sec-sur/2/ccarcs-riacc/download/ccarcsdb.zip + +The Government of Canada open licence requires both of the following notices, +verbatim, and requires that they reach the consumer together: + + Reproduced and distributed with the permission of the Government of Canada. + + This product has been produced by or for the OpenAirframes project and + includes data provided by the Government of Canada. The incorporation of + data sourced from the Government of Canada within this product shall not be + construed as constituting an endorsement by the Government of Canada of our + product. + +Owner mailing addresses published in the CCARCS export (street, city, postal +code, care-of) are dropped during ingestion and are not redistributed. + + +FAA — Releasable Aircraft Database +----------------------------------- +Asset: openairframes_faa_*.csv, ReleasableAircraft_*.zip +Source: https://registry.faa.gov/database/ReleasableAircraft.zip + +A work of the United States federal government, not subject to copyright +protection in the United States (17 U.S.C. § 105). No notice is required; the +source is credited here for provenance. + + +Other redistributed assets +--------------------------- +The following assets are republished from third parties whose terms have not +been assessed in this repository. They are listed for provenance only; nothing +here asserts a licence over them. + + openairframes_adsb_*.csv.gz derived from adsb.lol daily globe history + (github.com/adsblol), with registration data + from tar1090-db + basic-ac-db_*.json.gz ADS-B Exchange, downloads.adsbexchange.com + mictronics-db_*.zip Mictronics, www.mictronics.de + +Community submissions under community/ are contributed by their authors through +the repository's submission workflow and are published with the attribution each +contributor selected. From 2711b2d0f82292ed29c36feb76b074bed37d451b Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Fri, 28 Aug 2026 23:57:27 -0400 Subject: [PATCH 2/9] feat: build a daily Transport Canada aircraft register release - parse the headerless latin1 CCARCS export against its declared column layout - derive transponder_code_hex from the 24-bit Mode S binary, populated for all 34,913 rows - expand marks to C- and vintage CF- registrations and drop owner mailing addresses - mirror the FAA build: same concat-with-latest-release dedup and output conventions Generated-by: Claude Opus 5 --- src/create_daily_tc_release.py | 56 +++++++++++ src/derive_from_tc_ccarcs.py | 172 +++++++++++++++++++++++++++++++++ src/get_latest_release.py | 37 +++++++ 3 files changed, 265 insertions(+) create mode 100644 src/create_daily_tc_release.py create mode 100644 src/derive_from_tc_ccarcs.py diff --git a/src/create_daily_tc_release.py b/src/create_daily_tc_release.py new file mode 100644 index 0000000..912d83a --- /dev/null +++ b/src/create_daily_tc_release.py @@ -0,0 +1,56 @@ +from pathlib import Path +from datetime import datetime, timezone +import argparse + +parser = argparse.ArgumentParser(description="Create daily Transport Canada release") +parser.add_argument("--date", type=str, help="Date to process (YYYY-MM-DD format, default: today)") +args = parser.parse_args() + +if args.date: + date_str = args.date +else: + date_str = datetime.now(timezone.utc).strftime("%Y-%m-%d") + +out_dir = Path("data/tc_ccarcs") +out_dir.mkdir(parents=True, exist_ok=True) +zip_name = f"ccarcsdb_{date_str}.zip" + +zip_path = out_dir / zip_name +if not zip_path.exists(): + url = "https://wwwapps.tc.gc.ca/saf-sec-sur/2/ccarcs-riacc/download/ccarcsdb.zip" + from urllib.request import Request, urlopen + + # CCARCS rejects default urllib agents. + req = Request( + url, + headers={ + "User-Agent": ( + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36" + ) + }, + method="GET", + ) + + with urlopen(req, timeout=120) as r: + body = r.read() + zip_path.write_bytes(body) + +OUT_ROOT = Path("data/openairframes") +OUT_ROOT.mkdir(parents=True, exist_ok=True) +from derive_from_tc_ccarcs import convert_tc_ccarcs_to_df +# Row-fingerprint dedup is source-agnostic; reused rather than forked. +from derive_from_faa_master_txt import concat_faa_historical_df +from get_latest_release import get_latest_aircraft_tc_csv_df +df_new = convert_tc_ccarcs_to_df(zip_path, date_str) + +try: + df_base, start_date_str = get_latest_aircraft_tc_csv_df() + df_base = concat_faa_historical_df(df_base, df_new) + assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing" +except Exception as e: + print(f"No existing Transport Canada release found, using only new data: {e}") + df_base = df_new + start_date_str = date_str + +df_base.to_csv(OUT_ROOT / f"openairframes_tc_{start_date_str}_{date_str}.csv", index=False) diff --git a/src/derive_from_tc_ccarcs.py b/src/derive_from_tc_ccarcs.py new file mode 100644 index 0000000..31f68be --- /dev/null +++ b/src/derive_from_tc_ccarcs.py @@ -0,0 +1,172 @@ +from pathlib import Path +import csv +import io +import zipfile + +import pandas as pd + +from derive_from_faa_master_txt import normalize + +# CCARCS ships headerless, latin1, comma-delimited exports. Column names come from +# carslayout.txt in the same archive and must stay in file order. +CARSCURR_COLUMNS = [ + "MARK", "REGISTRATION_SUB_TYPE_E", "REGISTRATION_SUB_TYPE_F", "COMMON_NAME", + "MODEL_NAME", "MANUFACTURERS_SERIAL_NUMBER", "MANUFACTURER_SERIAL_COMPRESSED", + "ID_PLATE_MANUFACTURERS_NAME", "BASIS_FOR_REGISTRATION", "BASIS_FOR_REGISTRATION_F", + "AIRCRAFT_CATEGORY_E", "AIRCRAFT_CATEGORY_F", "DATE_OF_IMPORT", "ENGINE_MANUF", + "POWERGLIDER_FLAG", "ENGINE_CATEGORY_E", "ENGINE_CATEGORY_F", "NUMBER_OF_ENGINES", + "NUMBER_OF_SEATS", "AIR_WEIGHT_KILOS", "SALE_REPORTED", "ISSUE_DATE", + "EFFECTIVE_DATE", "INEFFECTIVE_DATE", "REGISTERED_PURPOSE_E", "REGISTERED_PURPOSE_F", + "FLIGHT_AUTHORITY_E", "FLIGHT_AUTHORITY_F", "MANUFACTURE_OR_ASSEMBLY", + "COUNTRY_MANUFACTURE_ASS_E", "COUNTRY_MANUFACTURE_ASS_F", "DATE_MANUFACTURE_ASSEMBLY", + "BASE_OF_OPERATIONS_CTRY_E", "BASE_OF_OPERATIONS_CTRY_F", "BASE_PROVINCE_OR_STATE_E", + "BASE_PROVINCE_OR_STATE_F", "CITY_AIRPORT", "TYPE_CERTIFICATE_NUMBER", + "REGISTRATION_AUTH_STATUS_E", "REGISTRATION_AUTH_STATUS_F", "MULTIPLE_OWNER_FLAG", + "MODIFIED_DATE", "MODE_S_TRANSPONDER_BINARY", "PHYSICAL_FILE_REGION_E", + "PHYSICAL_FILE_REGION_F", "EX_MILITARY_MARK", "TRIMMED_MARK", +] + +CARSOWNR_COLUMNS = [ + "MARK_LINK", "FULL_NAME", "TRADE_NAME", "STREET_NAME", "STREET_NAME2", "CITY", + "PROVINCE_OR_STATE_E", "PROVINCE_OR_STATE_F", "POSTAL_CODE", "COUNTRY_E", "COUNTRY_F", + "TYPE_OF_OWNER_E", "TYPE_OF_OWNER_F", "ACTIVE_FLAG", "CARE_OF", "REGION_E", "REGION_F", + "OWNER_NAME_OLD_FORMAT", "MAIL_RECIPIENT", "TRIMMED_MARK", +] + +# Owner mailing addresses are dropped rather than republished; see NOTICE. +OWNER_PII_COLUMNS = ["STREET_NAME", "STREET_NAME2", "CITY", "POSTAL_CODE", "CARE_OF"] + + +def _read_ccarcs_entry(zip_path: Path, entry: str, columns: list[str]) -> pd.DataFrame: + """Read one headerless CCARCS export into a DataFrame, dropping the Oracle footer. + + The export ends with a bare "N rows selected." line and a blank line; both are + narrower than the declared column count. Any *other* width mismatch is silent + field loss, so it raises instead. + """ + with zipfile.ZipFile(zip_path) as z: + text = z.read(entry).decode("latin1") + + rows = [] + ragged = 0 + for row in csv.reader(io.StringIO(text)): + if len(row) == len(columns): + rows.append([cell.strip() for cell in row]) + elif len(row) <= 1: + ragged += 1 # footer or trailing blank + else: + raise ValueError( + f"{entry}: row with {len(row)} fields, expected {len(columns)}" + ) + + if ragged > 2: + raise ValueError(f"{entry}: {ragged} ragged rows, expected at most 2") + + return pd.DataFrame(rows, columns=columns) + + +def tc_full_registration(mark: str) -> str: + """Expand a trimmed CCARCS mark into the full Canadian registration. + + Three-character marks are vintage CF- registrations; everything else takes the + modern C- prefix. + """ + mark = (mark or "").strip().upper() + if not mark: + return "" + return f"CF-{mark}" if len(mark) == 3 else f"C-{mark}" + + +def binary_to_hex(binary: str) -> str: + """Convert a 24-bit Mode S binary string to uppercase hex.""" + binary = (binary or "").strip() + if not binary or any(c not in "01" for c in binary): + return "" + return f"{int(binary, 2):06X}" + + +def _merge_owners(df_ownr: pd.DataFrame) -> pd.DataFrame: + """Collapse one row per registered party into one row per mark. + + A co-owned mark repeats with a different party each time; keeping only the mail + recipient would silently drop the rest. Each field is deduplicated independently + and blanks are skipped, so values are not index-parallel across columns. + """ + def join_unique(series: pd.Series) -> str: + seen = [] + for value in series: + value = (value or "").strip() + if value and value not in seen: + seen.append(value) + return ", ".join(seen) + + grouped = df_ownr.groupby("TRIMMED_MARK", sort=False).agg( + owner_name=("FULL_NAME", join_unique), + owner_province_or_state=("PROVINCE_OR_STATE_E", join_unique), + owner_country=("COUNTRY_E", join_unique), + owner_type=("TYPE_OF_OWNER_E", join_unique), + owner_party_count=("FULL_NAME", "size"), + ).reset_index() + + # A party row states its own type ("Individual"); that stops being true of the + # mark once several parties share it. + grouped.loc[grouped["owner_party_count"] > 1, "owner_type"] = "Co-owner" + return grouped + + +def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame: + """Build the OpenAirframes Transport Canada frame from a CCARCS zip.""" + df = _read_ccarcs_entry(zip_path, "carscurr.txt", CARSCURR_COLUMNS) + df_ownr = _read_ccarcs_entry(zip_path, "carsownr.txt", CARSOWNR_COLUMNS) + df_ownr = df_ownr.drop(columns=OWNER_PII_COLUMNS) + + df = df.merge(_merge_owners(df_ownr), on="TRIMMED_MARK", how="left") + + out = pd.DataFrame({ + "download_date": date, + "transponder_code_hex": df["MODE_S_TRANSPONDER_BINARY"].map(binary_to_hex), + "registration_number": df["TRIMMED_MARK"].map(tc_full_registration), + "mark": df["TRIMMED_MARK"], + "aircraft_manufacturer": df["COMMON_NAME"], + "aircraft_model": df["MODEL_NAME"], + "serial_number": df["MANUFACTURERS_SERIAL_NUMBER"], + "aircraft_category": df["AIRCRAFT_CATEGORY_E"], + "engine_manufacturer": df["ENGINE_MANUF"], + "engine_category": df["ENGINE_CATEGORY_E"], + "number_of_engines": df["NUMBER_OF_ENGINES"], + "number_of_seats": df["NUMBER_OF_SEATS"], + "max_weight_kilos": df["AIR_WEIGHT_KILOS"], + "registration_status": df["REGISTRATION_AUTH_STATUS_E"], + "registration_sub_type": df["REGISTRATION_SUB_TYPE_E"], + "basis_for_registration": df["BASIS_FOR_REGISTRATION"], + "registered_purpose": df["REGISTERED_PURPOSE_E"], + "flight_authority": df["FLIGHT_AUTHORITY_E"], + "type_certificate_number": df["TYPE_CERTIFICATE_NUMBER"], + "country_manufacture": df["COUNTRY_MANUFACTURE_ASS_E"], + "date_manufacture_assembly": df["DATE_MANUFACTURE_ASSEMBLY"], + "base_country": df["BASE_OF_OPERATIONS_CTRY_E"], + "base_province_or_state": df["BASE_PROVINCE_OR_STATE_E"], + "city_airport": df["CITY_AIRPORT"], + "ex_military_mark": df["EX_MILITARY_MARK"], + "multiple_owner_flag": df["MULTIPLE_OWNER_FLAG"], + "owner_name": df["owner_name"], + "owner_type": df["owner_type"], + "owner_province_or_state": df["owner_province_or_state"], + "owner_country": df["owner_country"], + "issue_date": df["ISSUE_DATE"], + "effective_date": df["EFFECTIVE_DATE"], + "ineffective_date": df["INEFFECTIVE_DATE"], + "modified_date": df["MODIFIED_DATE"], + }) + + out.insert(3, "openairframes_id", ( + normalize(out["aircraft_manufacturer"]) + + "|" + + normalize(out["aircraft_model"]) + + "|" + + normalize(out["serial_number"]) + )) + + out = out.fillna("") + out = out.replace("None", "") + return out diff --git a/src/get_latest_release.py b/src/get_latest_release.py index 27a2eca..d201f74 100644 --- a/src/get_latest_release.py +++ b/src/get_latest_release.py @@ -167,6 +167,43 @@ def get_latest_aircraft_faa_csv_df(): return df, date_str +def download_latest_aircraft_tc_csv( + output_dir: Path = Path("downloads"), + github_token: Optional[str] = None, + repo: str = REPO, +) -> Path: + """ + Download the latest openairframes_tc_*.csv file from the latest GitHub release. + + Args: + output_dir: Directory to save the downloaded file (default: "downloads") + github_token: Optional GitHub token for authentication + repo: GitHub repository in format "owner/repo" (default: REPO) + + Returns: + Path to the downloaded file + """ + output_dir = Path(output_dir) + assets = get_latest_release_assets(repo, github_token=github_token) + asset = pick_asset(assets, name_regex=r"^openairframes_tc_.*\.csv$") + saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token) + print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}") + return saved_to + + +def get_latest_aircraft_tc_csv_df(): + csv_path = download_latest_aircraft_tc_csv() + import pandas as pd + df = pd.read_csv(csv_path, dtype=str) + df = df.fillna("") + # Filename pattern: openairframes_tc_{start_date}_{end_date}.csv + match = re.search(r"openairframes_tc_(\d{4}-\d{2}-\d{2})_", str(csv_path)) + if not match: + raise ValueError(f"Could not extract date from filename: {csv_path.name}") + + return df, match.group(1) + + def download_latest_aircraft_adsb_csv( output_dir: Path = Path("downloads"), github_token: Optional[str] = None, From 705a97690f4f65678e59a5c5cacb4c18f95fb545 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Fri, 28 Aug 2026 23:57:27 -0400 Subject: [PATCH 3/9] docs: record the attribution and redistribution rules in AGENTS.md - attribution is a licence condition and NOTICE must travel with release assets - a public licence travels to this project; a bilateral permission does not - non-commercial-only registries are incompatible with the MIT-licensed releases Generated-by: Claude Opus 5 --- AGENTS.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index b523458..5534525 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,8 +9,9 @@ Never fix this class of bug by escaping, sanitizing, or renaming the delimiter ## Run everything from the repo root -`src/create_daily_faa_release.py` must be invoked as a **script** (`python src/create_daily_faa_release.py`). -It uses bare sibling imports, so `-m src.create_daily_faa_release` raises `ModuleNotFoundError`. +`src/create_daily_faa_release.py` and `src/create_daily_tc_release.py` must be invoked as **scripts** +(`python src/create_daily_faa_release.py`). They use bare sibling imports, so `-m` raises +`ModuleNotFoundError`. Everything under `src/adsb/` and `src/contributions/` is the opposite — `python -m`, package-relative. Output paths are CWD-relative. @@ -42,6 +43,24 @@ matrices. Reason about the YAML statically. - HTTP 404 is terminal in the release fetch. Restoring the retry makes the Dec-31 next-year-repo probe stall ~45 minutes on a repo that does not exist yet. +## Attribution is a licence condition, not a courtesy + +`NOTICE` carries the terms that make redistributable sources redistributable, and it is uploaded as +a release asset so it travels with the data. Deleting or editing an entry removes the permission for +the corresponding asset. + +Transport Canada requires **both** its notices — reproduction and value-added — to reach the +consumer together. `NOTICE` must also survive the `create-release` sparse checkout; it is listed +there explicitly. + +Before adding any registry, check redistribution, not just access. A public licence (CC BY, an +open-government licence) travels to this project; a bilateral permission granted to a different +project does not. Non-commercial-only sources are incompatible with the MIT-licensed releases — +that rules out Taiwan, Estonia and Chile even though they are cleared for private use elsewhere. + +Owner mailing addresses in the CCARCS export are dropped during ingestion; only name, province and +country are published. + ## Fork and upstream `src/get_latest_release.py` pins `REPO = "PlaneQuery/openairframes"` on purpose: this fork reads From f5423724bc5f00585f497bf101246718c72c5fae Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Mon, 31 Aug 2026 20:40:04 -0400 Subject: [PATCH 4/9] fix: validate the CCARCS parse and correct owner aggregation - reject Mode S fields that are not 24 binary digits; a short field zero-padded into a plausible address belonging to a different aircraft - check the "N rows selected." footer against the parsed row count and enforce a row floor, so an upstream short export fails instead of publishing as a smaller register - prefer ACTIVE_FLAG "A" parties but fall back to all: 1,932 Registered marks carry only "I" rows, and those are the MAIL_RECIPIENT, so filtering on "A" alone drops real owners - count distinct owner names rather than rows, fixing 154 marks labelled Co-owner in error - match the spool footer by pattern instead of a brittle ragged-row count Generated-by: Claude Opus 5 --- src/derive_from_tc_ccarcs.py | 95 ++++++++++++++++++++++++++---------- 1 file changed, 70 insertions(+), 25 deletions(-) diff --git a/src/derive_from_tc_ccarcs.py b/src/derive_from_tc_ccarcs.py index 31f68be..d89d4c6 100644 --- a/src/derive_from_tc_ccarcs.py +++ b/src/derive_from_tc_ccarcs.py @@ -1,6 +1,7 @@ from pathlib import Path import csv import io +import re import zipfile import pandas as pd @@ -37,30 +38,49 @@ OWNER_PII_COLUMNS = ["STREET_NAME", "STREET_NAME2", "CITY", "POSTAL_CODE", "CARE_OF"] +FOOTER_RE = re.compile(r"\s*(\d+) rows selected\.\s*") + +# Canada's register is ~35k aircraft. Any parse yielding less than this means the +# export was truncated upstream, which must not be published as a real snapshot. +MIN_EXPECTED_ROWS = 1000 + + def _read_ccarcs_entry(zip_path: Path, entry: str, columns: list[str]) -> pd.DataFrame: - """Read one headerless CCARCS export into a DataFrame, dropping the Oracle footer. + """Read one headerless CCARCS export into a DataFrame. - The export ends with a bare "N rows selected." line and a blank line; both are - narrower than the declared column count. Any *other* width mismatch is silent - field loss, so it raises instead. + Raises: + ValueError: on any row whose width is neither the declared column count nor a + blank/footer line, on a missing or disagreeing "N rows selected." footer, or + on a row count below MIN_EXPECTED_ROWS. """ with zipfile.ZipFile(zip_path) as z: text = z.read(entry).decode("latin1") rows = [] - ragged = 0 - for row in csv.reader(io.StringIO(text)): + declared = None + # newline="" so a CRLF export does not leave \r on the final field of every row. + for row in csv.reader(io.StringIO(text, newline="")): if len(row) == len(columns): rows.append([cell.strip() for cell in row]) - elif len(row) <= 1: - ragged += 1 # footer or trailing blank - else: - raise ValueError( - f"{entry}: row with {len(row)} fields, expected {len(columns)}" - ) - - if ragged > 2: - raise ValueError(f"{entry}: {ragged} ragged rows, expected at most 2") + continue + if not row or not any(cell.strip() for cell in row): + continue # trailing blank line + match = FOOTER_RE.fullmatch(row[0]) if len(row) == 1 else None + if match: + declared = int(match.group(1)) + continue + raise ValueError( + f"{entry}: row with {len(row)} fields, expected {len(columns)}: {row[:3]!r}" + ) + + # The spool footer is a free checksum from the source; a short export is otherwise + # indistinguishable from a genuinely smaller register. + if declared is None: + raise ValueError(f"{entry}: no 'N rows selected.' footer; export is truncated") + if declared != len(rows): + raise ValueError(f"{entry}: footer declares {declared} rows, parsed {len(rows)}") + if len(rows) < MIN_EXPECTED_ROWS: + raise ValueError(f"{entry}: only {len(rows)} rows, expected >= {MIN_EXPECTED_ROWS}") return pd.DataFrame(rows, columns=columns) @@ -68,8 +88,9 @@ def _read_ccarcs_entry(zip_path: Path, entry: str, columns: list[str]) -> pd.Dat def tc_full_registration(mark: str) -> str: """Expand a trimmed CCARCS mark into the full Canadian registration. - Three-character marks are vintage CF- registrations; everything else takes the - modern C- prefix. + CCARCS stores the bare mark in both MARK and TRIMMED_MARK, so the prefix has to be + reconstructed: three-character marks are vintage CF- registrations, everything else + takes the modern C- prefix. Returns "" for a blank mark. """ mark = (mark or "").strip().upper() if not mark: @@ -78,20 +99,38 @@ def tc_full_registration(mark: str) -> str: def binary_to_hex(binary: str) -> str: - """Convert a 24-bit Mode S binary string to uppercase hex.""" + """Convert a 24-bit Mode S binary string to a 6-digit uppercase hex address. + + Returns "" for empty, non-binary, or non-24-bit input. Width is checked because + this column is the join key against ADS-B data: a short field would otherwise + zero-pad into a plausible address belonging to a different aircraft. + """ binary = (binary or "").strip() - if not binary or any(c not in "01" for c in binary): + if len(binary) != 24 or any(c not in "01" for c in binary): return "" return f"{int(binary, 2):06X}" def _merge_owners(df_ownr: pd.DataFrame) -> pd.DataFrame: - """Collapse one row per registered party into one row per mark. + """Collapse the active registered parties for each mark into a single row. A co-owned mark repeats with a different party each time; keeping only the mail - recipient would silently drop the rest. Each field is deduplicated independently - and blanks are skipped, so values are not index-parallel across columns. + recipient would silently drop the rest. + + Each field is deduplicated and blank-skipped independently, so the values are NOT + index-parallel: a mark with three owners can emit three names but one province. + Consumers must not split on ", " and zip the columns together. """ + # ACTIVE_FLAG is "A"/"I", but "I" does not mean "former owner": 1,932 currently + # Registered marks carry only "I" parties, and those rows are the MAIL_RECIPIENT. + # So prefer active parties where a mark has any, and fall back to all of them + # rather than publishing a registered aircraft with no owner at all. + active = df_ownr[df_ownr["ACTIVE_FLAG"].str.upper() == "A"] + marks_with_active = set(active["TRIMMED_MARK"]) + df_ownr = pd.concat([ + active, + df_ownr[~df_ownr["TRIMMED_MARK"].isin(marks_with_active)], + ]) def join_unique(series: pd.Series) -> str: seen = [] for value in series: @@ -100,16 +139,20 @@ def join_unique(series: pd.Series) -> str: seen.append(value) return ", ".join(seen) + def count_distinct(series: pd.Series) -> int: + return len({v.strip() for v in series if v and v.strip()}) + grouped = df_ownr.groupby("TRIMMED_MARK", sort=False).agg( owner_name=("FULL_NAME", join_unique), owner_province_or_state=("PROVINCE_OR_STATE_E", join_unique), owner_country=("COUNTRY_E", join_unique), owner_type=("TYPE_OF_OWNER_E", join_unique), - owner_party_count=("FULL_NAME", "size"), + owner_party_count=("FULL_NAME", count_distinct), ).reset_index() - # A party row states its own type ("Individual"); that stops being true of the - # mark once several parties share it. + # A party row states its own type ("Individual"); that stops being true of the mark + # once several parties share it. Counting distinct names rather than rows keeps this + # consistent with owner_name, which is also deduplicated. grouped.loc[grouped["owner_party_count"] > 1, "owner_type"] = "Co-owner" return grouped @@ -159,6 +202,8 @@ def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame: "modified_date": df["MODIFIED_DATE"], }) + # Position matches the FAA frame (after registration_number). Ordering is cosmetic: + # concat_faa_historical_df reindexes df_new to the base's columns before merging. out.insert(3, "openairframes_id", ( normalize(out["aircraft_manufacturer"]) + "|" From 9a1b828d3c5177e4dbcc1bd0995e5f0570c5853c Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Mon, 31 Aug 2026 20:40:04 -0400 Subject: [PATCH 5/9] fix: stop a failed Transport Canada build from erasing release history - fall back to a single-day rebuild only on FileNotFoundError; a rate limit, schema change or truncated download previously took the same path and republished one day as the whole dataset, which the next run then read back as its base - move the monotonic download_date assert out of the try so corruption cannot select the destructive branch - walk back through releases like the ADS-B reader, so one missing optional asset does not strand the accumulation - authenticate release reads and verify downloaded asset size - read the previous CSV with keep_default_na=False so literal NA values round-trip - write the download atomically and reject non-zip responses Generated-by: Claude Opus 5 --- src/create_daily_tc_release.py | 29 +++++++++++++++++++----- src/get_latest_release.py | 41 ++++++++++++++++++++++++++++------ 2 files changed, 57 insertions(+), 13 deletions(-) diff --git a/src/create_daily_tc_release.py b/src/create_daily_tc_release.py index 912d83a..ecfc508 100644 --- a/src/create_daily_tc_release.py +++ b/src/create_daily_tc_release.py @@ -20,7 +20,8 @@ url = "https://wwwapps.tc.gc.ca/saf-sec-sur/2/ccarcs-riacc/download/ccarcsdb.zip" from urllib.request import Request, urlopen - # CCARCS rejects default urllib agents. + # CCARCS 403s a default urllib agent. Any browser-like UA works; the exact + # version string is not load-bearing. req = Request( url, headers={ @@ -34,23 +35,39 @@ with urlopen(req, timeout=120) as r: body = r.read() - zip_path.write_bytes(body) + # TC serves an HTML maintenance page with a 200, which would otherwise be cached + # under a .zip name and re-read on every later run. + if body[:2] != b"PK": + raise RuntimeError(f"{url} did not return a zip (got {body[:40]!r})") + tmp_path = zip_path.with_suffix(".part") + tmp_path.write_bytes(body) + tmp_path.replace(zip_path) OUT_ROOT = Path("data/openairframes") OUT_ROOT.mkdir(parents=True, exist_ok=True) from derive_from_tc_ccarcs import convert_tc_ccarcs_to_df -# Row-fingerprint dedup is source-agnostic; reused rather than forked. +# Named for FAA but column-agnostic: fingerprints every column except download_date. from derive_from_faa_master_txt import concat_faa_historical_df from get_latest_release import get_latest_aircraft_tc_csv_df df_new = convert_tc_ccarcs_to_df(zip_path, date_str) +# Only a genuine first run may rebuild from a single day. Every other failure -- a rate +# limit, a schema change, a truncated download -- must stop the run, because this file +# becomes tomorrow's base and silently republishing one day erases the whole history. try: df_base, start_date_str = get_latest_aircraft_tc_csv_df() +except FileNotFoundError as e: + print(f"No existing Transport Canada release found, bootstrapping from today only: {e}") + df_base = None + start_date_str = date_str + +if df_base is not None: + missing = set(df_base.columns) ^ set(df_new.columns) + if missing: + raise SystemExit(f"Column set changed since the last release: {sorted(missing)}") df_base = concat_faa_historical_df(df_base, df_new) assert df_base['download_date'].is_monotonic_increasing, "download_date is not monotonic increasing" -except Exception as e: - print(f"No existing Transport Canada release found, using only new data: {e}") +else: df_base = df_new - start_date_str = date_str df_base.to_csv(OUT_ROOT / f"openairframes_tc_{start_date_str}_{date_str}.csv", index=False) diff --git a/src/get_latest_release.py b/src/get_latest_release.py index d201f74..0161d2e 100644 --- a/src/get_latest_release.py +++ b/src/get_latest_release.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Iterable, Optional +import os import re import urllib.request import urllib.error @@ -184,19 +185,45 @@ def download_latest_aircraft_tc_csv( Path to the downloaded file """ output_dir = Path(output_dir) - assets = get_latest_release_assets(repo, github_token=github_token) - asset = pick_asset(assets, name_regex=r"^openairframes_tc_.*\.csv$") - saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token) - print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}") - return saved_to + github_token = github_token or os.environ.get("GITHUB_TOKEN") + + # Walk back through releases rather than reading only `latest`. The TC asset is + # optional, so a single failed build publishes a release without it; anchoring on + # `latest` would then make the caller rebuild history from one day and republish + # that as the whole dataset. + for release in get_releases(repo, github_token=github_token, per_page=30): + assets = get_release_assets_from_release_data(release) + try: + asset = pick_asset(assets, name_regex=r"^openairframes_tc_.*\.csv$") + except FileNotFoundError: + continue + saved_to = download_asset(asset, output_dir / asset.name, github_token=github_token) + if asset.size and saved_to.stat().st_size != asset.size: + raise RuntimeError( + f"{asset.name}: downloaded {saved_to.stat().st_size} bytes, expected {asset.size}" + ) + print(f"Downloaded: {asset.name} ({asset.size} bytes) -> {saved_to}") + return saved_to + + raise FileNotFoundError( + "No release in the last 30 releases has an asset matching 'openairframes_tc_.*\\.csv$'" + ) def get_latest_aircraft_tc_csv_df(): + """Return (DataFrame, start_date_str) for the most recent published TC release. + + Raises FileNotFoundError when no recent release carries a TC asset, and ValueError + when the asset filename has no parseable start date. + """ csv_path = download_latest_aircraft_tc_csv() import pandas as pd - df = pd.read_csv(csv_path, dtype=str) + # keep_default_na=False: a literal "NA"/"N/A" in the source would otherwise read back + # as NaN -> "" while the fresh parse keeps the string, so the row fingerprints would + # never match and every affected record would re-append on every run. + df = pd.read_csv(csv_path, dtype=str, keep_default_na=False) df = df.fillna("") - # Filename pattern: openairframes_tc_{start_date}_{end_date}.csv + # Only the start date is taken; the end date is always the run's own date. match = re.search(r"openairframes_tc_(\d{4}-\d{2}-\d{2})_", str(csv_path)) if not match: raise ValueError(f"Could not extract date from filename: {csv_path.name}") From bd78cc5b2ce62feb14a19ce1f4c9905a6c2866e4 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Mon, 31 Aug 2026 20:40:04 -0400 Subject: [PATCH 6/9] docs: correct licence and ownership claims in NOTICE and AGENTS.md - stop asserting US public-domain status for the derived FAA CSV; section 105 covers the government's own work, not this repository's derivative - name no licence instrument for Transport Canada, which does not publish one on the download page, rather than citing one that cannot be verified - state that MIT covers code only, resolving a contradiction with AGENTS.md - lead the source-eligibility rule with the bilateral-permission bar; Taiwan is OGDL licensed and excluded for that reason, not for commercial terms Generated-by: Claude Opus 5 --- AGENTS.md | 32 ++++++++++++++++++++------------ NOTICE | 21 ++++++++++++++------- 2 files changed, 34 insertions(+), 19 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5534525..9745ffd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -43,22 +43,30 @@ matrices. Reason about the YAML statically. - HTTP 404 is terminal in the release fetch. Restoring the retry makes the Dec-31 next-year-repo probe stall ~45 minutes on a repo that does not exist yet. -## Attribution is a licence condition, not a courtesy +## Never edit or drop a `NOTICE` entry -`NOTICE` carries the terms that make redistributable sources redistributable, and it is uploaded as -a release asset so it travels with the data. Deleting or editing an entry removes the permission for -the corresponding asset. +Each entry is the permission that makes its asset redistributable; removing one removes the +permission. `NOTICE` is validated as a **required** release file and is listed in the +`create-release` sparse checkout — keep both. Transport Canada requires its two notices, +reproduction and value-added, to reach the consumer together. -Transport Canada requires **both** its notices — reproduction and value-added — to reach the -consumer together. `NOTICE` must also survive the `create-release` sparse checkout; it is listed -there explicitly. +`LICENSE` is MIT and covers **code only**. It makes no claim over released data, and neither may +you — an asset derived from a public-domain source is not itself public domain. -Before adding any registry, check redistribution, not just access. A public licence (CC BY, an -open-government licence) travels to this project; a bilateral permission granted to a different -project does not. Non-commercial-only sources are incompatible with the MIT-licensed releases — -that rules out Taiwan, Estonia and Chile even though they are cleared for private use elsewhere. +Before adding any registry, judge redistribution, not access, in this order: -Owner mailing addresses in the CCARCS export are dropped during ingestion; only name, province and +1. **A public licence travels; a bilateral permission does not.** Written permission granted to + another project or person is not a licence to this one. That alone disqualifies Taiwan, Estonia + and Chile, whatever their commercial terms say. +2. **Non-commercial-only conditions are a second, independent bar** — they conflict with how these + releases are consumed. Do not treat a source clearing this bar as cleared overall; rule 1 still + applies. Taiwan is licensed OGDL v1.0, an open licence: its restriction is bilateral, not licence-borne. + +CCARCS `ACTIVE_FLAG` does **not** mean "current owner": 1,932 currently-Registered marks carry only +`I` parties, and those rows are the `MAIL_RECIPIENT`. Prefer `A` parties where a mark has any, fall +back to all of them otherwise. Filtering on `A` alone publishes registered aircraft with no owner. + +Owner mailing addresses (street, city, postal code, care-of) are dropped; name, type, province and country are published. ## Fork and upstream diff --git a/NOTICE b/NOTICE index 22726d7..d038cd1 100644 --- a/NOTICE +++ b/NOTICE @@ -4,7 +4,8 @@ OpenAirframes — Data Source Notices The LICENSE file covers the *code* in this repository. It does not cover the *data* published in releases. Several upstream registries permit redistribution only on condition that specific notices travel with the data. Those conditions -are reproduced below and must ship alongside any redistributed release asset. +are reproduced below. This file is published as an asset on every release; anyone +redistributing a release asset further must carry the corresponding notice with it. Removing or altering a notice in this file removes the permission that makes the corresponding asset redistributable. @@ -15,8 +16,10 @@ Transport Canada — Canadian Civil Aircraft Register (CCARCS) Asset: openairframes_tc_*.csv Source: https://wwwapps.tc.gc.ca/saf-sec-sur/2/ccarcs-riacc/download/ccarcsdb.zip -The Government of Canada open licence requires both of the following notices, -verbatim, and requires that they reach the consumer together: +Redistribution is permitted under the Government of Canada terms recorded for this +dataset. They require both of the following notices, verbatim, and require that the +two reach the consumer together. The governing instrument is not published on the +CCARCS download page, so no licence name or URL is asserted here. Reproduced and distributed with the permission of the Government of Canada. @@ -32,12 +35,16 @@ code, care-of) are dropped during ingestion and are not redistributed. FAA — Releasable Aircraft Database ----------------------------------- -Asset: openairframes_faa_*.csv, ReleasableAircraft_*.zip Source: https://registry.faa.gov/database/ReleasableAircraft.zip -A work of the United States federal government, not subject to copyright -protection in the United States (17 U.S.C. § 105). No notice is required; the -source is credited here for provenance. +ReleasableAircraft_*.zip is redistributed unmodified. It is a work of the United +States federal government, not subject to copyright protection in the United +States (17 U.S.C. § 105). No notice is required; it is credited for provenance. + +openairframes_faa_*.csv is a derived product built by this repository — +normalized, joined, deduplicated, and extended with an identifier this project +defines. Section 105 disclaims copyright in the government's own work and says +nothing about a derivative, so no claim is made here about the CSV's status. Other redistributed assets From fce0b8d18cc7f3b0cfc53779b2007837e5f022c1 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Mon, 31 Aug 2026 20:55:29 -0400 Subject: [PATCH 7/9] feat: redistribute Canadian owner addresses to match the FAA asset - publish street, city, postal code and care-of, which the FAA asset already carries as registrant_* for 99.7% of US registrants; dropping them here left one repository with two different postures on the same class of data - take the address from the single MAIL_RECIPIENT row rather than merging across parties, since a co-owned mark lists several people in different cities Generated-by: Claude Opus 5 --- AGENTS.md | 6 ++++-- NOTICE | 6 ++++-- src/derive_from_tc_ccarcs.py | 33 +++++++++++++++++++++++++++++---- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9745ffd..6ff20e8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -66,8 +66,10 @@ CCARCS `ACTIVE_FLAG` does **not** mean "current owner": 1,932 currently-Register `I` parties, and those rows are the `MAIL_RECIPIENT`. Prefer `A` parties where a mark has any, fall back to all of them otherwise. Filtering on `A` alone publishes registered aircraft with no owner. -Owner mailing addresses (street, city, postal code, care-of) are dropped; name, type, province and -country are published. +Owner mailing addresses **are** published, matching the `registrant_*` address the FAA asset already +carries — the two sources must not diverge on this. Addresses are per-party, so they come from the +single `MAIL_RECIPIENT == "Y"` row (exactly one per mark) and are never merged across co-owners; only +name, type, province and country are merged lists. ## Fork and upstream diff --git a/NOTICE b/NOTICE index d038cd1..fcfffde 100644 --- a/NOTICE +++ b/NOTICE @@ -29,8 +29,10 @@ CCARCS download page, so no licence name or URL is asserted here. construed as constituting an endorsement by the Government of Canada of our product. -Owner mailing addresses published in the CCARCS export (street, city, postal -code, care-of) are dropped during ingestion and are not redistributed. +Registered-owner mailing addresses are redistributed, matching the registrant +addresses the FAA asset already carries. Because CCARCS lists one row per party, +the published address is that of the single designated mail recipient rather than +a merge across co-owners. FAA — Releasable Aircraft Database diff --git a/src/derive_from_tc_ccarcs.py b/src/derive_from_tc_ccarcs.py index d89d4c6..1cc9f7d 100644 --- a/src/derive_from_tc_ccarcs.py +++ b/src/derive_from_tc_ccarcs.py @@ -34,8 +34,16 @@ "OWNER_NAME_OLD_FORMAT", "MAIL_RECIPIENT", "TRIMMED_MARK", ] -# Owner mailing addresses are dropped rather than republished; see NOTICE. -OWNER_PII_COLUMNS = ["STREET_NAME", "STREET_NAME2", "CITY", "POSTAL_CODE", "CARE_OF"] +# Mailing address of the single designated recipient, matching the registrant_* address +# the FAA build already publishes. Addresses are per-party, so they are taken from the one +# MAIL_RECIPIENT row rather than merged across co-owners. +OWNER_ADDRESS_COLUMNS = { + "STREET_NAME": "owner_street_1", + "STREET_NAME2": "owner_street_2", + "CITY": "owner_city", + "POSTAL_CODE": "owner_postal_code", + "CARE_OF": "owner_care_of", +} FOOTER_RE = re.compile(r"\s*(\d+) rows selected\.\s*") @@ -125,6 +133,7 @@ def _merge_owners(df_ownr: pd.DataFrame) -> pd.DataFrame: # Registered marks carry only "I" parties, and those rows are the MAIL_RECIPIENT. # So prefer active parties where a mark has any, and fall back to all of them # rather than publishing a registered aircraft with no owner at all. + all_parties = df_ownr active = df_ownr[df_ownr["ACTIVE_FLAG"].str.upper() == "A"] marks_with_active = set(active["TRIMMED_MARK"]) df_ownr = pd.concat([ @@ -154,14 +163,25 @@ def count_distinct(series: pd.Series) -> int: # once several parties share it. Counting distinct names rather than rows keeps this # consistent with owner_name, which is also deduplicated. grouped.loc[grouped["owner_party_count"] > 1, "owner_type"] = "Co-owner" - return grouped + + # Taken from the unfiltered frame: the designated recipient is the designated + # recipient even when its own party row is flagged inactive. + recipient = ( + all_parties[all_parties["MAIL_RECIPIENT"].str.upper() == "Y"] + .drop_duplicates(subset="TRIMMED_MARK", keep="first") + .rename(columns=OWNER_ADDRESS_COLUMNS) + ) + return grouped.merge( + recipient[["TRIMMED_MARK", *OWNER_ADDRESS_COLUMNS.values()]], + on="TRIMMED_MARK", + how="left", + ) def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame: """Build the OpenAirframes Transport Canada frame from a CCARCS zip.""" df = _read_ccarcs_entry(zip_path, "carscurr.txt", CARSCURR_COLUMNS) df_ownr = _read_ccarcs_entry(zip_path, "carsownr.txt", CARSOWNR_COLUMNS) - df_ownr = df_ownr.drop(columns=OWNER_PII_COLUMNS) df = df.merge(_merge_owners(df_ownr), on="TRIMMED_MARK", how="left") @@ -196,6 +216,11 @@ def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame: "owner_type": df["owner_type"], "owner_province_or_state": df["owner_province_or_state"], "owner_country": df["owner_country"], + "owner_care_of": df["owner_care_of"], + "owner_street_1": df["owner_street_1"], + "owner_street_2": df["owner_street_2"], + "owner_city": df["owner_city"], + "owner_postal_code": df["owner_postal_code"], "issue_date": df["ISSUE_DATE"], "effective_date": df["EFFECTIVE_DATE"], "ineffective_date": df["INEFFECTIVE_DATE"], From 1b6de19afb63d277c3da3f483a284acb30b182e2 Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Mon, 31 Aug 2026 21:12:22 -0400 Subject: [PATCH 8/9] refactor: name Transport Canada columns in the FAA vocabulary - registrant_* rather than owner_*, status rather than registration_status, so both registries describe the same concept with the same column name - registrant_zip_code carries the Canadian postal code: a union table needs one column per concept, not one per country's vocabulary - set source="TC", matching the discriminator the FAA frame already carries - raises the column names shared with the FAA frame from 9 to 21 Generated-by: Claude Opus 5 --- src/derive_from_tc_ccarcs.py | 50 +++++++++++++++++++----------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/src/derive_from_tc_ccarcs.py b/src/derive_from_tc_ccarcs.py index 1cc9f7d..510f2c5 100644 --- a/src/derive_from_tc_ccarcs.py +++ b/src/derive_from_tc_ccarcs.py @@ -37,12 +37,14 @@ # Mailing address of the single designated recipient, matching the registrant_* address # the FAA build already publishes. Addresses are per-party, so they are taken from the one # MAIL_RECIPIENT row rather than merged across co-owners. +# registrant_zip_code holds the Canadian postal code: the name is the FAA's, and a union +# table needs one column per concept, not one per country's vocabulary. OWNER_ADDRESS_COLUMNS = { - "STREET_NAME": "owner_street_1", - "STREET_NAME2": "owner_street_2", - "CITY": "owner_city", - "POSTAL_CODE": "owner_postal_code", - "CARE_OF": "owner_care_of", + "STREET_NAME": "registrant_street_1", + "STREET_NAME2": "registrant_street_2", + "CITY": "registrant_city", + "POSTAL_CODE": "registrant_zip_code", + "CARE_OF": "registrant_care_of", } @@ -152,17 +154,17 @@ def count_distinct(series: pd.Series) -> int: return len({v.strip() for v in series if v and v.strip()}) grouped = df_ownr.groupby("TRIMMED_MARK", sort=False).agg( - owner_name=("FULL_NAME", join_unique), - owner_province_or_state=("PROVINCE_OR_STATE_E", join_unique), - owner_country=("COUNTRY_E", join_unique), - owner_type=("TYPE_OF_OWNER_E", join_unique), - owner_party_count=("FULL_NAME", count_distinct), + registrant_name=("FULL_NAME", join_unique), + registrant_state=("PROVINCE_OR_STATE_E", join_unique), + registrant_country=("COUNTRY_E", join_unique), + registrant_type=("TYPE_OF_OWNER_E", join_unique), + registrant_party_count=("FULL_NAME", count_distinct), ).reset_index() # A party row states its own type ("Individual"); that stops being true of the mark # once several parties share it. Counting distinct names rather than rows keeps this # consistent with owner_name, which is also deduplicated. - grouped.loc[grouped["owner_party_count"] > 1, "owner_type"] = "Co-owner" + grouped.loc[grouped["registrant_party_count"] > 1, "registrant_type"] = "Co-owner" # Taken from the unfiltered frame: the designated recipient is the designated # recipient even when its own party row is flagged inactive. @@ -187,6 +189,8 @@ def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame: out = pd.DataFrame({ "download_date": date, + # The FAA frame already carries `source`; it is the union discriminator. + "source": "TC", "transponder_code_hex": df["MODE_S_TRANSPONDER_BINARY"].map(binary_to_hex), "registration_number": df["TRIMMED_MARK"].map(tc_full_registration), "mark": df["TRIMMED_MARK"], @@ -196,10 +200,10 @@ def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame: "aircraft_category": df["AIRCRAFT_CATEGORY_E"], "engine_manufacturer": df["ENGINE_MANUF"], "engine_category": df["ENGINE_CATEGORY_E"], - "number_of_engines": df["NUMBER_OF_ENGINES"], - "number_of_seats": df["NUMBER_OF_SEATS"], + "aircraft_number_of_engines": df["NUMBER_OF_ENGINES"], + "aircraft_number_of_seats": df["NUMBER_OF_SEATS"], "max_weight_kilos": df["AIR_WEIGHT_KILOS"], - "registration_status": df["REGISTRATION_AUTH_STATUS_E"], + "status": df["REGISTRATION_AUTH_STATUS_E"], "registration_sub_type": df["REGISTRATION_SUB_TYPE_E"], "basis_for_registration": df["BASIS_FOR_REGISTRATION"], "registered_purpose": df["REGISTERED_PURPOSE_E"], @@ -212,15 +216,15 @@ def convert_tc_ccarcs_to_df(zip_path: Path, date: str) -> pd.DataFrame: "city_airport": df["CITY_AIRPORT"], "ex_military_mark": df["EX_MILITARY_MARK"], "multiple_owner_flag": df["MULTIPLE_OWNER_FLAG"], - "owner_name": df["owner_name"], - "owner_type": df["owner_type"], - "owner_province_or_state": df["owner_province_or_state"], - "owner_country": df["owner_country"], - "owner_care_of": df["owner_care_of"], - "owner_street_1": df["owner_street_1"], - "owner_street_2": df["owner_street_2"], - "owner_city": df["owner_city"], - "owner_postal_code": df["owner_postal_code"], + "registrant_name": df["registrant_name"], + "registrant_type": df["registrant_type"], + "registrant_state": df["registrant_state"], + "registrant_country": df["registrant_country"], + "registrant_care_of": df["registrant_care_of"], + "registrant_street_1": df["registrant_street_1"], + "registrant_street_2": df["registrant_street_2"], + "registrant_city": df["registrant_city"], + "registrant_zip_code": df["registrant_zip_code"], "issue_date": df["ISSUE_DATE"], "effective_date": df["EFFECTIVE_DATE"], "ineffective_date": df["INEFFECTIVE_DATE"], From 2c5e8b962129858358565e1c9f6efb39b095a62b Mon Sep 17 00:00:00 2001 From: Ashley Childress Date: Mon, 31 Aug 2026 21:12:22 -0400 Subject: [PATCH 9/9] docs: cut AGENTS.md back to operative constraints - drop narrative rationale, restated rules and background a reader can grep for - one rule per line; the file is read by a model about to act, not by a person catching up - 151 lines to 85 Generated-by: Claude Opus 5 --- AGENTS.md | 202 ++++++++++++++++++------------------------------------ 1 file changed, 67 insertions(+), 135 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 6ff20e8..1dc9c76 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,150 +1,82 @@ -## Never interpolate `${{ github.event.* }}` into a `run:` block +## Never interpolate `${{ github.event.* }}` or `${{ inputs.* }}` into a `run:` block -Actions substitutes `${{ }}` as raw text before the shell parses it, and issue bodies here are -public and unauthenticated. Pass untrusted values through `env:` and quote them. +Pass them via `env:` and quote the shell variable. A quoted heredoc does not help — the body can +contain the delimiter and close it early. Never fix this by escaping or renaming the delimiter. -A quoted heredoc delimiter does not save you: the body can *contain* the delimiter, close the -heredoc early, and execute every line after it. `validate-community-submission.yaml` had that shape. -Never fix this class of bug by escaping, sanitizing, or renaming the delimiter — move it to `env:`. +## Invocation -## Run everything from the repo root - -`src/create_daily_faa_release.py` and `src/create_daily_tc_release.py` must be invoked as **scripts** -(`python src/create_daily_faa_release.py`). They use bare sibling imports, so `-m` raises -`ModuleNotFoundError`. -Everything under `src/adsb/` and `src/contributions/` is the opposite — `python -m`, package-relative. - -Output paths are CWD-relative. +- `src/*.py` at the root of `src/` are scripts: `python src/create_daily_faa_release.py`. Bare + sibling imports, so `-m` raises `ModuleNotFoundError`. +- `src/adsb/*`, `src/contributions/*` are packages: `python -m`. +- Run from the repo root; output paths are CWD-relative. ## Verification -There is no test framework, linter, or packaging config. **Do not add one unprompted**, and do not -treat "nothing broke" as verification. - -The ADS-B path has no cheap end-to-end check — one day of input is tens of GB. Exercise -`compress_multi_icao_df` / `compress_df_polars` directly against a small hand-built Polars frame. - -**Never `gh workflow run` to test a change.** Every dispatch pulls tens of GB and fans out over date -matrices. Reason about the YAML statically. - -**Never commit generated data.** The product is a GitHub Release; jobs pass state as artifacts. - -## ADS-B invariants - -- `FINAL_COLUMN_ORDER` (`compress_adsb_to_aircraft_data.py`) is the only definition of the released - column contract. `pl.concat` matches by **position** after `.select()`, and - `get_latest_release.get_latest_aircraft_adsb_csv_df` parses released CSVs against the same order — - so a forked copy corrupts the release with no error anywhere. -- `load_parquet_part()` deleting its source parquet is **deliberate**: the raw part is many GB and - the runner would otherwise exhaust disk. Do not defer the delete to make reruns easier; that raises - peak disk by the size of the part. -- A released row means "most informative observation for this ICAO on this UTC day" — non-empty - fields not a subset of another row's, tie-broken by signature frequency. It is not a registry record. -- HTTP 404 is terminal in the release fetch. Restoring the retry makes the Dec-31 next-year-repo - probe stall ~45 minutes on a repo that does not exist yet. - -## Never edit or drop a `NOTICE` entry - -Each entry is the permission that makes its asset redistributable; removing one removes the -permission. `NOTICE` is validated as a **required** release file and is listed in the -`create-release` sparse checkout — keep both. Transport Canada requires its two notices, -reproduction and value-added, to reach the consumer together. - -`LICENSE` is MIT and covers **code only**. It makes no claim over released data, and neither may -you — an asset derived from a public-domain source is not itself public domain. - -Before adding any registry, judge redistribution, not access, in this order: - -1. **A public licence travels; a bilateral permission does not.** Written permission granted to - another project or person is not a licence to this one. That alone disqualifies Taiwan, Estonia - and Chile, whatever their commercial terms say. -2. **Non-commercial-only conditions are a second, independent bar** — they conflict with how these - releases are consumed. Do not treat a source clearing this bar as cleared overall; rule 1 still - applies. Taiwan is licensed OGDL v1.0, an open licence: its restriction is bilateral, not licence-borne. - -CCARCS `ACTIVE_FLAG` does **not** mean "current owner": 1,932 currently-Registered marks carry only -`I` parties, and those rows are the `MAIL_RECIPIENT`. Prefer `A` parties where a mark has any, fall -back to all of them otherwise. Filtering on `A` alone publishes registered aircraft with no owner. - -Owner mailing addresses **are** published, matching the `registrant_*` address the FAA asset already -carries — the two sources must not diverge on this. Addresses are per-party, so they come from the -single `MAIL_RECIPIENT == "Y"` row (exactly one per mark) and are never merged across co-owners; only -name, type, province and country are merged lists. - -## Fork and upstream - -`src/get_latest_release.py` pins `REPO = "PlaneQuery/openairframes"` on purpose: this fork reads -**upstream's** releases wherever it runs. Do not repoint it at `github.repository` without being asked. - -Upstream develops on `develop` and PRs into `main`. The daily release **deletes the existing release -and tag** before recreating them. +- No test framework, linter, or packaging config. Do not add one unprompted. +- Never `gh workflow run` to test a change — every dispatch pulls tens of GB. +- Never commit generated data. The product is a GitHub Release; jobs pass state as artifacts. +- ADS-B has no cheap end-to-end check. Exercise `compress_multi_icao_df` on a hand-built frame. + +## Release invariants + +- `FINAL_COLUMN_ORDER` (`compress_adsb_to_aircraft_data.py`) is the only definition of the ADS-B + column contract. `pl.concat` matches by position after `.select()`; a forked copy corrupts the + release with no error. +- Empty string, never null, in every released frame. +- `openairframes_id` = `normalize(manufacturer)|normalize(model)|normalize(serial)`. Reuse + `derive_from_faa_master_txt.normalize()`. +- Each source's daily build reads its own previous release asset and appends. Falling back to a + single-day rebuild on anything but `FileNotFoundError` republishes one day as the whole dataset, + which the next run then reads back as its base. Keep the fallback narrow. +- Python `3.14` for FAA/community/vendor jobs, `3.12` for ADS-B. Match the surrounding job. + +## Registry sources + +- Judge **redistribution**, not access. A public licence travels to this project; a bilateral + permission granted to another project does not — that alone disqualifies Taiwan, Estonia, Chile. + Non-commercial-only terms are a separate, independent bar. +- `NOTICE` carries the terms that make each asset redistributable and is a required release file. + Never edit or drop an entry. Transport Canada requires both its notices together. +- `LICENSE` is MIT and covers code only. Claim nothing about released data. +- Owner/registrant mailing addresses are published for every registry. FAA and TC must not diverge. +- CCARCS `ACTIVE_FLAG` is not "current owner": 1,932 Registered marks carry only `I` parties, and + those rows are the `MAIL_RECIPIENT`. Prefer `A`, fall back to all. +- CCARCS addresses come from the single `MAIL_RECIPIENT == "Y"` row, never merged across co-owners. + +## ADS-B + +- `load_parquet_part()` deleting its source parquet is deliberate — disk pressure. Do not defer it. +- A released row is the most informative observation for that ICAO on that UTC day, not a registry + record. +- HTTP 404 is terminal in the release fetch; restoring the retry stalls the Dec-31 probe ~45 min. ## Community submissions are automation-owned -Merging to `community/**` or `schemas/**` force-pushes every open `community`-labeled PR branch back -onto main. Anything you hand-edit on such a branch is destroyed on the next merge. - -- Never hand-author files in `community/` — the filename encodes `sha256(content)[:8]`, so an edit - orphans the hash and duplicates on re-approval. -- Never invent or copy a `contributor_uuid`; it is derived from the GitHub user id. -- Do not reintroduce a hardcoded `"main"` or `v1` filename. Both are resolved at runtime now. - -**A tag's JSON type is fixed by its first-ever submission and enforced forever** — emergent from -`build_tag_type_registry` + `validate_submission`, and written nowhere in the schema. Retyping or -renaming an existing tag breaks every future contributor, not just the current one. - -Dropping a `community_submission.v2.schema.json` into `schemas/` promotes it atomically across every -reader and writer. That is a one-way door for contributors — only on explicit request. - -## Conventions - -- **Empty string, not null**, everywhere in released frames. -- Reuse `derive_from_faa_master_txt.normalize()` for `openairframes_id`; never re-derive the format. -- Python `3.14` for FAA/community/vendor jobs, `3.12` for ADS-B jobs (pyarrow pin + multiprocessing). - Deliberate. Match the surrounding job; do not unify. - -## References with no target — do not chase as regressions - -| Reference | Missing | -|---|---| -| `process-historical-faa.yaml` | `src/get_historical_faa.py`, `scripts/concat_csvs.py` | -| `af-klm-fleet/package.json` → `npm run validate` | `af-klm-fleet/scripts/validate.js` | - -`process-historical-faa.yaml` is dead, not stale — it also uses the disabled `::set-output`. -Repair-vs-delete is the owner's call; leave it alone unprompted. - -## `af-klm-fleet/` and `community-routes/` are unwired - -Nothing in CI touches either, and nothing consumes `community-routes/`. `af-klm-fleet/` is a vendored -project by a different author with its own license — its aircraft model is unrelated to -`schemas/community_submission.*`, so do not merge the two. Its `README.md` is generated by -`generate-readme.js`; hand edits are overwritten. - -## Warts left standing — flag, do not silently fix +- Merging `community/**` or `schemas/**` force-pushes every open `community` PR branch onto main. + Hand edits there are destroyed. +- Never hand-author `community/` files — the filename encodes `sha256(content)[:8]`. +- A tag's JSON type is fixed by its first submission and enforced forever. Emergent from + `build_tag_type_registry` + `validate_submission`; written nowhere in the schema. +- Adding `community_submission.v2.schema.json` promotes it atomically across every reader and + writer. One-way door — only on request. -- `NUMBER_PARTS` is restated by the matrix and four hand-written upload steps in - `adsb-to-aircraft-for-day.yaml`; changing the constant alone silently drops data. YAML cannot loop - upload steps and one merged artifact would force every map job to download all parts, so any real - fix is a restructure. -- `MAX_WORKERS = OS_CPU_COUNT if OS_CPU_COUNT > 4 else 1` collapses to a single worker on a ≤4-core - runner, shrinking `files_per_batch` with it. Possibly intentional memory control — do not raise it - without measuring peak RSS on the target runner. -- `update-community-prs.yaml` runs `regenerate_pr_schema || true` then force-pushes, so a - regeneration failure ships anyway. Making it fatal leaves PRs un-rebased instead — a judgment call. -- `approve_submission.py` wraps its schema update in a bare `except Exception`, so a submission can - merge without its new tags reaching the schema. +## Fork -## Workflow authoring +`get_latest_release.REPO` pins upstream `PlaneQuery/openairframes` on purpose. Do not repoint it. +Upstream develops on `develop`. The daily release deletes the existing release and tag first. -The user's global GitHub Actions rules apply. Existing workflows violate most of them. -**Do not bulk-remediate** — bring only the file you were asked to touch up to standard, and surface -the rest in chat. +## Do not chase -## External sources +- `process-historical-faa.yaml` is dead: missing `src/get_historical_faa.py`, + `scripts/concat_csvs.py`, and uses the disabled `::set-output`. +- `af-klm-fleet/package.json` → `npm run validate` has no `scripts/validate.js`. +- `af-klm-fleet/` and `community-routes/` are unwired; nothing in CI touches them. + `af-klm-fleet/README.md` is generated. -`registry.faa.gov` and ADS-B Exchange are **required** — the release fails without them. Mictronics is -**tolerated**: it retries, then the job continues without it. adsb.lol may simply not have published a -given day, in which case the previous CSV is re-released rather than failing. +## Flag, do not silently fix -FAA refreshes at 05:30 UTC; the release cron fires at 06:00 UTC. That 30-minute margin is the reason -for the schedule. +- `NUMBER_PARTS` is restated by the matrix and four upload steps in `adsb-to-aircraft-for-day.yaml`. +- `MAX_WORKERS = ... if OS_CPU_COUNT > 4 else 1` collapses to one worker on a ≤4-core runner. +- `update-community-prs.yaml` runs `regenerate_pr_schema || true`, then force-pushes. +- `approve_submission.py` wraps its schema update in a bare `except Exception`. +- Existing workflows violate the global GHA rules. Fix only the file you were asked to touch.