From 7490918ebc16f636e7e94362ea2de92df7854139 Mon Sep 17 00:00:00 2001 From: Vraj Patel <51841184+vrajpatell@users.noreply.github.com> Date: Fri, 26 Jun 2026 20:37:45 +0530 Subject: [PATCH] Fix earthquake source filtering and coordinate normalization --- .env.example | 19 +++-- .python-version | 2 +- README.md | 90 +++++++++++++++++++++ app/__init__.py | 7 ++ app/config.py | 10 ++- app/routes.py | 116 ++++++++++++++++++--------- app/services/cache.py | 39 ++++++--- app/services/sources/__init__.py | 1 + app/services/sources/auspass.py | 5 ++ app/services/sources/base.py | 25 ++++++ app/services/sources/bmkg.py | 24 ++++++ app/services/sources/emsc.py | 3 + app/services/sources/geofon.py | 5 ++ app/services/sources/geonet.py | 14 ++++ app/services/sources/isc.py | 5 ++ app/services/sources/normalizer.py | 47 +++++++++++ app/services/sources/registry.py | 20 +++++ app/services/sources/usgs_fdsn.py | 16 ++++ app/services/sources/usgs_geojson.py | 14 ++++ app/services/usgs_client.py | 2 +- static/js/dashboard.js | 2 +- static/js/forecast.js | 2 +- static/js/globe.js | 26 +++--- static/js/map.js | 2 +- templates/dashboard.html | 2 +- templates/forecast.html | 2 +- templates/globe.html | 4 +- templates/map.html | 2 +- tests/test_app.py | 39 +++++++++ 29 files changed, 470 insertions(+), 75 deletions(-) create mode 100644 app/services/sources/__init__.py create mode 100644 app/services/sources/auspass.py create mode 100644 app/services/sources/base.py create mode 100644 app/services/sources/bmkg.py create mode 100644 app/services/sources/emsc.py create mode 100644 app/services/sources/geofon.py create mode 100644 app/services/sources/geonet.py create mode 100644 app/services/sources/isc.py create mode 100644 app/services/sources/normalizer.py create mode 100644 app/services/sources/registry.py create mode 100644 app/services/sources/usgs_fdsn.py create mode 100644 app/services/sources/usgs_geojson.py diff --git a/.env.example b/.env.example index b5adcfc..48f0d8e 100644 --- a/.env.example +++ b/.env.example @@ -1,9 +1,12 @@ -USGS_FEED_URL=https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.geojson -CACHE_TTL_SECONDS=300 -SQLITE_DB_PATH=data/earthquakes.db -DEFAULT_FEED_WINDOW=day -MODEL_PATH=models/earthquake_risk.joblib -ENABLE_BOOTSTRAP_DATA=true -ENABLE_MODEL_TRAINING_ON_STARTUP=false +ENABLE_BOOTSTRAP_DATA=false +ALLOW_BOOTSTRAP_AS_FRESH_CACHE=false +ENABLED_SOURCES=usgs_geojson,usgs_fdsn,emsc,isc,geofon,bmkg,geonet +SOURCE_TIMEOUT_SECONDS=15 +SOURCE_MAX_RETRIES=2 +SOURCE_REFRESH_TTL_SECONDS=300 +SOURCE_BMKG_RATE_LIMIT_PER_MINUTE=60 +SOURCE_ENABLE_AUSPASS=false +SOURCE_ENABLE_EXPERIMENTAL=false MAX_API_LIMIT=2000 -ADMIN_TOKEN=change-me-for-admin-training +ADMIN_TOKEN=change-me +SECRET_KEY=change-me diff --git a/.python-version b/.python-version index ed7d51a..2c07333 100644 --- a/.python-version +++ b/.python-version @@ -1 +1 @@ -3.11.15 +3.11 diff --git a/README.md b/README.md index 6bf806e..a3d87df 100644 --- a/README.md +++ b/README.md @@ -97,3 +97,93 @@ python -m compileall . - Background scheduled ingestion. - Authentication for admin operations. - Screenshot automation for README assets. + +## Multi-source earthquake data and coordinate safety + +This Flask dashboard ingests public earthquake observations, normalizes every event into canonical `latitude` and `longitude` fields, stores them in SQLite, and serves the map, globe, dashboard, and probabilistic forecast from the same filtered API path. + +### Wrong-location bug fix + +The map/globe coordinate order was not globally swapped. USGS GeoJSON correctly uses `[longitude, latitude, depth]`, Leaflet correctly uses `[latitude, longitude]`, and globe.gl correctly uses `{ lat, lng }`. Wrong-looking points were primarily caused by random bootstrap demo records being mixed with real cached feeds, feed-specific cache results not being strictly filtered, bootstrap cache being treated as fresh data, and longitude bounds being sorted across the antimeridian. The API now excludes bootstrap records by default, filters by `source` and `source_feed`, tracks refresh metadata per source/feed, validates coordinates before storage, and handles dateline-crossing viewports with `longitude >= west OR longitude <= east`. + +### Coordinate normalization rules + +Different sources use different coordinate order. This app normalizes every source into canonical latitude/longitude fields before storage and visualization. + +* Backend canonical fields are always `latitude` and `longitude`. +* Leaflet receives `[latitude, longitude]`. +* globe.gl receives `{ lat: latitude, lng: longitude }`. +* USGS GeoJSON, USGS FDSN GeoJSON, and GeoNet GeoJSON parse `[longitude, latitude, depth]`. +* BMKG `Coordinates` parse as `latitude,longitude`. +* Invalid, NaN, infinite, out-of-range, or silently swapped-invalid coordinates are rejected before reaching the frontend. + +### Public data sources + +| Source | Coverage | Format | Role | +| --- | --- | --- | --- | +| USGS GeoJSON | Global | GeoJSON | Primary live source | +| USGS FDSN | Global | GeoJSON/CSV/XML | Historical and custom search | +| EMSC / SeismicPortal | Europe/global | FDSN JSON/QuakeML/CSV | Cross-checking | +| ISC | Global reviewed catalogue | FDSN | Historical/reviewed (stubbed if unavailable) | +| GFZ/GEOFON | Global/Europe | FDSN | Additional global source (stubbed if unavailable) | +| BMKG | Indonesia | JSON/XML | Asia/Indonesia | +| GeoNet | New Zealand | GeoJSON/JSON | Oceania/New Zealand | +| AusPass | Australia/Oceania | FDSN | Optional, disabled by default | + +### Attribution + +Data remains attributed in `/api/sources` and frontend status areas. Primary attribution includes USGS Earthquake Hazards Program, EMSC / SeismicPortal, BMKG Indonesia, GeoNet, ISC, GFZ/GEOFON, and AusPass where enabled. + +### Environment variables + +Recommended production settings: + +```text +ENABLE_BOOTSTRAP_DATA=false +ALLOW_BOOTSTRAP_AS_FRESH_CACHE=false +ENABLED_SOURCES=usgs_geojson,usgs_fdsn,emsc,isc,geofon,bmkg,geonet +SOURCE_TIMEOUT_SECONDS=15 +SOURCE_MAX_RETRIES=2 +SOURCE_REFRESH_TTL_SECONDS=300 +MAX_API_LIMIT=2000 +ADMIN_TOKEN=change-me +SECRET_KEY=change-me +``` + +### API examples + +```bash +curl '/api/earthquakes?source=usgs&feed=day&include_demo=false' +curl '/api/earthquakes?source=usgs&feed=30day&mag_min=4.5' +curl '/api/earthquakes?source=all&feed=day&lon1=170&lon2=-170' +curl '/api/sources' +curl -X POST '/api/refresh' -H 'Content-Type: application/json' -d '{"source":"all","feed":"day"}' +``` + +If demo records are explicitly requested with `include_demo=true`, the API and UI warn: “Demo data — not real earthquake locations.” + +### Render deployment + +Keep deployment simple: + +```bash +pip install -r requirements.txt +gunicorn main:app --bind 0.0.0.0:$PORT +``` + +Set `ENABLE_BOOTSTRAP_DATA=false` on Render so random bootstrap records are never displayed as real earthquake locations. + +### Local verification + +Run: + +```bash +pytest +python -m compileall . +``` + +Open `/map` and `/globe`, select a source/feed, and test a dateline viewport near Alaska, Japan, Fiji, New Zealand, Indonesia, or the Pacific. + +### Scientific disclaimer + +The forecast page is probabilistic and educational. It summarizes recent regional seismicity and model-derived risk indicators; it does not predict exact earthquakes, times, or locations. diff --git a/app/__init__.py b/app/__init__.py index 3e986b4..57dd1d4 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -9,6 +9,13 @@ def create_app(config_class=Config): app=Flask(__name__, template_folder='../templates', static_folder='../static') app.config.from_object(config_class) + # Re-read environment-backed settings at app creation time so tests and deployments + # can override values after the Config class has been imported. + import os + if os.getenv('SQLITE_DB_PATH'): app.config['SQLITE_DB_PATH']=os.getenv('SQLITE_DB_PATH') + if os.getenv('ENABLE_BOOTSTRAP_DATA') is not None: app.config['ENABLE_BOOTSTRAP_DATA']=os.getenv('ENABLE_BOOTSTRAP_DATA','false').lower() in {'1','true','yes','on'} + if os.getenv('ALLOW_BOOTSTRAP_AS_FRESH_CACHE') is not None: app.config['ALLOW_BOOTSTRAP_AS_FRESH_CACHE']=os.getenv('ALLOW_BOOTSTRAP_AS_FRESH_CACHE','false').lower() in {'1','true','yes','on'} + if os.getenv('ENABLED_SOURCES'): app.config['ENABLED_SOURCES']=os.getenv('ENABLED_SOURCES') logging.basicConfig(level=logging.INFO) init_db(app.config['SQLITE_DB_PATH']) seed_bootstrap_if_empty(app.config['SQLITE_DB_PATH'], app.config['ENABLE_BOOTSTRAP_DATA']) diff --git a/app/config.py b/app/config.py index ce1d492..c715d78 100644 --- a/app/config.py +++ b/app/config.py @@ -10,12 +10,20 @@ class Config: SQLITE_DB_PATH = os.getenv('SQLITE_DB_PATH', str(BASE_DIR / 'data' / 'earthquakes.db')) DEFAULT_FEED_WINDOW = os.getenv('DEFAULT_FEED_WINDOW', 'day') MODEL_PATH = os.getenv('MODEL_PATH', str(BASE_DIR / 'models' / 'earthquake_risk.joblib')) - ENABLE_BOOTSTRAP_DATA = os.getenv('ENABLE_BOOTSTRAP_DATA', 'true').lower() in {'1','true','yes','on'} + ENABLE_BOOTSTRAP_DATA = os.getenv('ENABLE_BOOTSTRAP_DATA', 'false').lower() in {'1','true','yes','on'} + ALLOW_BOOTSTRAP_AS_FRESH_CACHE = os.getenv('ALLOW_BOOTSTRAP_AS_FRESH_CACHE', 'false').lower() in {'1','true','yes','on'} ENABLE_MODEL_TRAINING_ON_STARTUP = os.getenv('ENABLE_MODEL_TRAINING_ON_STARTUP', 'false').lower() in {'1','true','yes','on'} MAX_API_LIMIT = int(os.getenv('MAX_API_LIMIT', '2000')) ADMIN_TOKEN = os.getenv('ADMIN_TOKEN', '') FLASK_ENV = os.getenv('FLASK_ENV', os.getenv('ENV', 'production')) REQUEST_TIMEOUT_SECONDS = int(os.getenv('REQUEST_TIMEOUT_SECONDS', '15')) + SOURCE_TIMEOUT_SECONDS = int(os.getenv('SOURCE_TIMEOUT_SECONDS', os.getenv('REQUEST_TIMEOUT_SECONDS', '15'))) + SOURCE_MAX_RETRIES = int(os.getenv('SOURCE_MAX_RETRIES', '2')) + SOURCE_REFRESH_TTL_SECONDS = int(os.getenv('SOURCE_REFRESH_TTL_SECONDS', os.getenv('CACHE_TTL_SECONDS', '300'))) + SOURCE_BMKG_RATE_LIMIT_PER_MINUTE = int(os.getenv('SOURCE_BMKG_RATE_LIMIT_PER_MINUTE', '60')) + SOURCE_ENABLE_AUSPASS = os.getenv('SOURCE_ENABLE_AUSPASS', 'false').lower() in {'1','true','yes','on'} + SOURCE_ENABLE_EXPERIMENTAL = os.getenv('SOURCE_ENABLE_EXPERIMENTAL', 'false').lower() in {'1','true','yes','on'} + ENABLED_SOURCES = os.getenv('ENABLED_SOURCES', 'usgs_geojson,usgs_fdsn,emsc,isc,geofon,bmkg,geonet') FEEDS = { 'hour': 'https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_hour.geojson', diff --git a/app/routes.py b/app/routes.py index 478299c..2d144d0 100644 --- a/app/routes.py +++ b/app/routes.py @@ -3,37 +3,69 @@ from flask import Blueprint, current_app, jsonify, render_template, request from app.services import analytics, model from app.services.cache import all_meta, all_records, connect, count_records, feed_meta_key, query_earthquakes, seed_bootstrap_if_empty, set_meta, upsert_earthquakes +from app.services.sources import SOURCE_REGISTRY, enabled_source_names, resolve_sources from app.services.usgs_client import fetch_feed from app.utils import bool_arg, parse_time_ms, safe_float, safe_int bp=Blueprint('main',__name__) def feed_url(name): - feeds=current_app.config['FEEDS']; return current_app.config['USGS_FEED_URL'] if name=='custom' else feeds.get(name, feeds.get(current_app.config['DEFAULT_FEED_WINDOW'], current_app.config['USGS_FEED_URL'])) -def feed_meta_value(meta, key, feed, default=''): - scoped=feed_meta_key(key,feed) - if scoped in meta: return meta[scoped] + feeds=current_app.config['FEEDS'] + return current_app.config['USGS_FEED_URL'] if name=='custom' else feeds.get(name, feeds.get(current_app.config['DEFAULT_FEED_WINDOW'], current_app.config['USGS_FEED_URL'])) + +def feed_meta_value(meta, key, feed, source='usgs', default=''): + keys=[feed_meta_key(key,feed,source), f'{key}:{feed}'] + for k in keys: + if k in meta: return meta[k] return meta.get(key, default) if meta.get('last_refresh_source') == feed else default -def refresh_if_needed(feed='day', force=False): - db=current_app.config['SQLITE_DB_PATH']; ttl=current_app.config['CACHE_TTL_SECONDS']; meta=all_meta(db) - last=int(feed_meta_value(meta,'last_successful_refresh_epoch',feed,'0') or 0) - if not force and int(time.time())-last <= ttl: return 'cache' - try: - rows=fetch_feed(feed_url(feed), current_app.config['REQUEST_TIMEOUT_SECONDS']); n=upsert_earthquakes(db,rows,feed,False) - return f'live:{n}' - except Exception as exc: - current_app.logger.warning('USGS refresh failed: %s', exc) - with connect(db) as c: - set_meta(c,'last_refresh_error',str(exc)[:300]); set_meta(c,'last_refresh_source','fallback_cache') - set_meta(c,feed_meta_key('last_refresh_error',feed),str(exc)[:300]); set_meta(c,feed_meta_key('last_refresh_source',feed),'fallback_cache') - c.commit() - if count_records(db)==0: seed_bootstrap_if_empty(db, current_app.config['ENABLE_BOOTSTRAP_DATA']); return 'bootstrap_fallback' - return 'fallback_cache' +def source_params_from_request(): + return {'timeout': current_app.config['SOURCE_TIMEOUT_SECONDS'], 'limit': request.args.get('limit'), + 'minmagnitude': request.args.get('min_magnitude') or request.args.get('mag_min'), + 'maxmagnitude': request.args.get('max_magnitude') or request.args.get('mag_max'), + 'starttime': request.args.get('start_time'), 'endtime': request.args.get('end_time'), + 'minlatitude': request.args.get('lat1'), 'maxlatitude': request.args.get('lat2'), + 'minlongitude': request.args.get('lon1'), 'maxlongitude': request.args.get('lon2'), + 'latitude': request.args.get('lat'), 'longitude': request.args.get('lon'), + 'maxradiuskm': request.args.get('radius_km')} + +def refresh_if_needed(feed='day', force=False, source='usgs'): + db=current_app.config['SQLITE_DB_PATH']; ttl=current_app.config.get('SOURCE_REFRESH_TTL_SECONDS', current_app.config['CACHE_TTL_SECONDS']); meta=all_meta(db) + sources=resolve_sources(source) + results=[] + for src in sources: + last=int(feed_meta_value(meta,'last_successful_refresh_epoch',feed,src.name,'0') or 0) + last_source=feed_meta_value(meta,'last_refresh_source',feed,src.name,'') + local_force=force or (last_source=='bootstrap_demo' and not current_app.config['ALLOW_BOOTSTRAP_AS_FRESH_CACHE']) + if not local_force and int(time.time())-last <= ttl: + results.append('cache'); continue + try: + rows=fetch_feed(feed_url(feed), current_app.config['REQUEST_TIMEOUT_SECONDS']) if src.name == 'usgs_geojson' else src.fetch(feed, source_params_from_request() if request else {'timeout': current_app.config['SOURCE_TIMEOUT_SECONDS']}) + n=upsert_earthquakes(db, rows, feed, False) + results.append(f'live:{n}' if src.name == 'usgs_geojson' and source in {'usgs','usgs_geojson'} else f'live:{src.name}:{n}') + except Exception as exc: + current_app.logger.warning('%s refresh failed: %s', src.name, exc) + with connect(db) as c: + set_meta(c,feed_meta_key('last_refresh_error',feed,src.name),str(exc)[:300]); set_meta(c,feed_meta_key('last_refresh_source',feed,src.name),'fallback_cache'); c.commit() + results.append('fallback_cache') + if results and all(r=='cache' for r in results): return 'cache' + if count_records(db)==0: + seed_bootstrap_if_empty(db, current_app.config['ENABLE_BOOTSTRAP_DATA']); return 'bootstrap_fallback' + return ','.join(results or ['cache']) + def filters_from_args(): - mag_min=safe_float(request.args.get('mag_min'),0.0); mag_max=safe_float(request.args.get('mag_max'),10.0) + mag_min=safe_float(request.args.get('mag_min') or request.args.get('min_magnitude'),0.0); mag_max=safe_float(request.args.get('mag_max') or request.args.get('max_magnitude'),10.0) lat1=safe_float(request.args.get('lat1'),-90.0); lat2=safe_float(request.args.get('lat2'),90.0); lon1=safe_float(request.args.get('lon1'),-180.0); lon2=safe_float(request.args.get('lon2'),180.0) - mag_min,mag_max=sorted((mag_min,mag_max)); lat1=max(-90,min(90,lat1)); lat2=max(-90,min(90,lat2)); lon1=max(-180,min(180,lon1)); lon2=max(-180,min(180,lon2)); lat_min,lat_max=sorted((lat1,lat2)); lon_min,lon_max=sorted((lon1,lon2)) - return {'mag_min':mag_min,'mag_max':mag_max,'lat_min':lat_min,'lat_max':lat_max,'lon_min':lon_min,'lon_max':lon_max,'start_time':parse_time_ms(request.args.get('start_time')),'end_time':parse_time_ms(request.args.get('end_time'))} + mag_min,mag_max=sorted((mag_min,mag_max)); lat1=max(-90,min(90,lat1)); lat2=max(-90,min(90,lat2)); lon1=max(-180,min(180,lon1)); lon2=max(-180,min(180,lon2)); lat_min,lat_max=sorted((lat1,lat2)) + crosses=lon1 > lon2 + return {'mag_min':mag_min,'mag_max':mag_max,'lat_min':lat_min,'lat_max':lat_max,'lon1':lon1,'lon2':lon2,'crosses_antimeridian':crosses,'start_time':parse_time_ms(request.args.get('start_time')),'end_time':parse_time_ms(request.args.get('end_time')),'source':request.args.get('source','usgs'),'source_feed':request.args.get('feed', current_app.config['DEFAULT_FEED_WINDOW']),'include_demo':bool_arg('include_demo')} + +def common_rows(do_refresh=True): + source=request.args.get('source','usgs'); feed=request.args.get('feed', current_app.config['DEFAULT_FEED_WINDOW']) + refresh=refresh_if_needed(feed,bool_arg('refresh'),source) if do_refresh else 'cache' + limit=safe_int(request.args.get('limit'),500,1,current_app.config['MAX_API_LIMIT']); sort=request.args.get('sort','time_desc') + filt=filters_from_args(); rows=query_earthquakes(current_app.config['SQLITE_DB_PATH'],filt,limit,sort) + return source, feed, refresh, filt, rows + @bp.route('/') @bp.route('/dashboard') def dashboard(): return render_template('dashboard.html') @@ -47,28 +79,38 @@ def forecast_page(): return render_template('forecast.html') def about(): return render_template('about.html') @bp.route('/api/earthquakes') def api_earthquakes(): - feed=request.args.get('feed', current_app.config['DEFAULT_FEED_WINDOW']); refresh=refresh_if_needed(feed,bool_arg('refresh')) - limit=safe_int(request.args.get('limit'),500,1,current_app.config['MAX_API_LIMIT']); sort=request.args.get('sort','time_desc') - filt=filters_from_args(); rows=query_earthquakes(current_app.config['SQLITE_DB_PATH'],filt,limit,sort); meta=all_meta(current_app.config['SQLITE_DB_PATH']) - warnings=[] - source=feed_meta_value(meta,'last_refresh_source',feed,'cache') - last_epoch=int(feed_meta_value(meta,'last_successful_refresh_epoch',feed,'0') or 0) - if 'fallback' in refresh or source in {'bootstrap_demo','fallback_cache'}: warnings.append('Using cached or bootstrap data because live USGS refresh was unavailable or skipped.') - return jsonify({'meta':{'source':source,'refresh_result':refresh,'last_refresh_epoch':last_epoch,'count':len(rows),'available_feeds':list(current_app.config['FEEDS'].keys())},'filters':filt,'count':len(rows),'source':source,'warnings':warnings,'data':rows}) + source,feed,refresh,filt,rows=common_rows(True); meta=all_meta(current_app.config['SQLITE_DB_PATH']) + warnings=[]; demo_count=sum(1 for r in rows if r.get('is_bootstrap')) + if demo_count: warnings.append('Demo data — not real earthquake locations.') + if 'fallback' in refresh: warnings.append('Using cached data because one or more live source refreshes failed.') + real_count=len(rows)-demo_count; last_epoch=max([int(feed_meta_value(meta,'last_successful_refresh_epoch',feed,s,'0') or 0) for s in enabled_source_names()] or [0]) + available_feeds=sorted({f for s in SOURCE_REGISTRY.values() for f in s.supported_feeds}) + return jsonify({'meta':{'source':(source if request.args.get('source') else feed),'requested_source':source,'feed':feed,'refresh_result':refresh,'last_refresh_epoch':last_epoch,'count':len(rows),'real_count':real_count,'demo_count':demo_count,'crosses_antimeridian':filt['crosses_antimeridian'],'available_sources':enabled_source_names(),'available_feeds':available_feeds,'warnings':warnings},'filters':filt,'count':len(rows),'source':source,'warnings':warnings,'data':rows}) +@bp.route('/api/sources') +def api_sources(): + meta=all_meta(current_app.config['SQLITE_DB_PATH']); enabled=set(enabled_source_names()); out=[] + for name,src in SOURCE_REGISTRY.items(): + if name=='usgs': continue + last=max([int(feed_meta_value(meta,'last_successful_refresh_epoch',f,name,'0') or 0) for f in src.supported_feeds] or [0]) + out.append({'name':name,'display_name':src.display_name,'enabled':name in enabled,'coverage':src.coverage,'last_refresh':last,'last_error':feed_meta_value(meta,'last_refresh_error',src.supported_feeds[0] if src.supported_feeds else 'day',name,None),'record_count':feed_meta_value(meta,'last_record_count',src.supported_feeds[0] if src.supported_feeds else 'day',name,'0'),'attribution':src.attribution,'homepage_url':src.homepage_url,'supported_feeds':src.supported_feeds}) + return jsonify({'sources':out}) +@bp.route('/api/refresh', methods=['POST']) +def api_refresh(): + token=current_app.config.get('ADMIN_TOKEN') + if token and request.headers.get('X-Admin-Token') != token: return jsonify({'error':'unauthorized'}), 403 + payload=request.get_json(silent=True) or {}; return jsonify({'refresh_result':refresh_if_needed(payload.get('feed','day'), True, payload.get('source','all'))}) @bp.route('/api/summary') def api_summary(): - refresh_if_needed(request.args.get('feed', current_app.config['DEFAULT_FEED_WINDOW']), False) - rows=query_earthquakes(current_app.config['SQLITE_DB_PATH'],filters_from_args(),current_app.config['MAX_API_LIMIT'],'time_desc') - feed=request.args.get('feed', current_app.config['DEFAULT_FEED_WINDOW']); meta=all_meta(current_app.config['SQLITE_DB_PATH']) - return jsonify({'summary':analytics.summarize(rows),'count':len(rows),'source':feed_meta_value(meta,'last_refresh_source',feed,'cache')}) + source,feed,refresh,filt,rows=common_rows(True); return jsonify({'summary':analytics.summarize(rows),'count':len(rows),'source':source,'meta':{'source':source,'feed':feed,'refresh_result':refresh}}) @bp.route('/api/regions') -def api_regions(): return jsonify({'regions':analytics.regions(query_earthquakes(current_app.config['SQLITE_DB_PATH'],filters_from_args(),current_app.config['MAX_API_LIMIT'],'time_desc'))}) +def api_regions(): source,feed,refresh,filt,rows=common_rows(True); return jsonify({'regions':analytics.regions(rows),'meta':{'source':source,'feed':feed}}) @bp.route('/api/timeseries') -def api_timeseries(): return jsonify({'interval':request.args.get('interval','hour'),'series':analytics.timeseries(query_earthquakes(current_app.config['SQLITE_DB_PATH'],filters_from_args(),current_app.config['MAX_API_LIMIT'],'time_desc'), request.args.get('interval','hour'))}) +def api_timeseries(): source,feed,refresh,filt,rows=common_rows(True); return jsonify({'interval':request.args.get('interval','hour'),'series':analytics.timeseries(rows, request.args.get('interval','hour')),'meta':{'source':source,'feed':feed}}) @bp.route('/api/forecast') def api_forecast(): + source,feed,refresh,filt,rows=common_rows(True) lat=safe_float(request.args.get('lat'),37.7749); lon=safe_float(request.args.get('lon'),-122.4194); radius=safe_float(request.args.get('radius_km'),300); hours=safe_int(request.args.get('hours'),72,1,24*30); minmag=safe_float(request.args.get('min_magnitude'),2.5) - rows=all_records(current_app.config['SQLITE_DB_PATH'], current_app.config['MAX_API_LIMIT']); return jsonify(model.predict(rows,current_app.config['MODEL_PATH'],lat,lon,radius,hours,minmag)) + result=model.predict(rows,current_app.config['MODEL_PATH'],lat,lon,radius,hours,minmag); result['meta']={'source':source,'feed':feed,'refresh_result':refresh}; return jsonify(result) @bp.route('/api/model/train', methods=['POST']) def api_train(): token=current_app.config.get('ADMIN_TOKEN') diff --git a/app/services/cache.py b/app/services/cache.py index b2099c2..e67a681 100644 --- a/app/services/cache.py +++ b/app/services/cache.py @@ -4,7 +4,7 @@ from typing import Any from app.utils import rough_region -COLUMNS=['id','time_ms','updated_ms','place','magnitude','longitude','latitude','depth','alert','tsunami','significance','felt','cdi','mmi','status','event_type','network','code','url','detail_url','mag_type','gap','dmin','rms','nst','region'] +COLUMNS=['id','source','time_ms','updated_ms','place','magnitude','longitude','latitude','depth','alert','tsunami','significance','felt','cdi','mmi','status','event_type','network','code','url','detail_url','mag_type','gap','dmin','rms','nst','region'] def connect(db_path: str) -> sqlite3.Connection: Path(db_path).parent.mkdir(parents=True, exist_ok=True) @@ -16,18 +16,22 @@ def connect(db_path: str) -> sqlite3.Connection: def init_db(db_path: str) -> None: with connect(db_path) as c: c.execute('''CREATE TABLE IF NOT EXISTS earthquakes ( - id TEXT PRIMARY KEY, time_ms INTEGER, updated_ms INTEGER, place TEXT, magnitude REAL, + id TEXT PRIMARY KEY, source TEXT DEFAULT 'usgs', time_ms INTEGER, updated_ms INTEGER, place TEXT, magnitude REAL, longitude REAL, latitude REAL, depth REAL, alert TEXT, tsunami INTEGER, significance INTEGER, felt INTEGER, cdi REAL, mmi REAL, status TEXT, event_type TEXT, network TEXT, code TEXT, url TEXT, detail_url TEXT, mag_type TEXT, gap REAL, dmin REAL, rms REAL, nst INTEGER, region TEXT, ingested_at INTEGER, source_feed TEXT, is_bootstrap INTEGER DEFAULT 0)''') + + existing={r['name'] for r in c.execute('PRAGMA table_info(earthquakes)').fetchall()} + if 'source' not in existing: c.execute("ALTER TABLE earthquakes ADD COLUMN source TEXT DEFAULT 'usgs'") + if 'raw_source' not in existing: c.execute("ALTER TABLE earthquakes ADD COLUMN raw_source TEXT") c.execute('''CREATE TABLE IF NOT EXISTS refresh_metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)''') - for idx,col in [('idx_quakes_time','time_ms'),('idx_quakes_mag','magnitude'),('idx_quakes_lat','latitude'),('idx_quakes_lon','longitude'),('idx_quakes_region','region')]: + for idx,col in [('idx_quakes_time','time_ms'),('idx_quakes_mag','magnitude'),('idx_quakes_lat','latitude'),('idx_quakes_lon','longitude'),('idx_quakes_region','region'),('idx_quakes_source','source'),('idx_quakes_feed','source_feed'),('idx_quakes_bootstrap','is_bootstrap')]: c.execute(f'CREATE INDEX IF NOT EXISTS {idx} ON earthquakes({col})') c.commit() -def feed_meta_key(k: str, feed: str) -> str: - return f'{k}:{feed}' +def feed_meta_key(k: str, feed: str, source: str='usgs') -> str: + return f'{k}:{source}:{feed}' def set_meta(c, k, v): c.execute('INSERT INTO refresh_metadata(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value',(k,str(v))) def get_meta(db_path: str, k: str, default: str='') -> str: @@ -41,14 +45,17 @@ def upsert_earthquakes(db_path: str, rows: list[dict[str,Any]], source_feed='unk now=int(time.time()) vals=[] for r in rows: - r=dict(r); r.setdefault('region', rough_region(r.get('latitude'), r.get('longitude'), r.get('place',''))) + r=dict(r); r.setdefault('source', 'bootstrap_demo' if is_bootstrap else ('usgs' if str(source_feed) in {'hour','day','7day','30day','significant','m1','m2.5','m4.5'} else str(r.get('source') or 'usgs'))); r.setdefault('region', rough_region(r.get('latitude'), r.get('longitude'), r.get('place',''))) vals.append(tuple(r.get(col) for col in COLUMNS)+(now,source_feed,1 if is_bootstrap else 0)) ph=','.join(['?']*(len(COLUMNS)+3)) updates=','.join([f'{c}=excluded.{c}' for c in COLUMNS[1:]]+['ingested_at=excluded.ingested_at','source_feed=excluded.source_feed','is_bootstrap=excluded.is_bootstrap']) with connect(db_path) as c: c.executemany(f'INSERT INTO earthquakes({",".join(COLUMNS)},ingested_at,source_feed,is_bootstrap) VALUES({ph}) ON CONFLICT(id) DO UPDATE SET {updates}', vals) - set_meta(c,'last_successful_refresh_epoch',now); set_meta(c,'last_refresh_source',source_feed); set_meta(c,'last_record_count',len(rows)) - set_meta(c,feed_meta_key('last_successful_refresh_epoch',source_feed),now); set_meta(c,feed_meta_key('last_refresh_source',source_feed),source_feed); set_meta(c,feed_meta_key('last_record_count',source_feed),len(rows)); set_meta(c,feed_meta_key('last_refresh_error',source_feed),'') + source_name = ('bootstrap_demo' if is_bootstrap else ((rows[0].get('source') if rows else None) or ('usgs' if str(source_feed) in {'hour','day','7day','30day','significant','m1','m2.5','m4.5'} else 'usgs'))) + set_meta(c,'last_successful_refresh_epoch',now); set_meta(c,'last_refresh_source',source_name); set_meta(c,'last_record_count',len(rows)) + set_meta(c,feed_meta_key('last_successful_refresh_epoch',source_feed,source_name),now); set_meta(c,feed_meta_key('last_refresh_source',source_feed,source_name),source_name); set_meta(c,feed_meta_key('last_record_count',source_feed,source_name),len(rows)); set_meta(c,feed_meta_key('last_refresh_error',source_feed,source_name),'') + # legacy feed-only metadata for backward compatibility + set_meta(c,f'last_successful_refresh_epoch:{source_feed}',now); set_meta(c,f'last_refresh_source:{source_feed}',source_feed); set_meta(c,f'last_record_count:{source_feed}',len(rows)); set_meta(c,f'last_refresh_error:{source_feed}','') c.commit() return len(rows) @@ -61,15 +68,27 @@ def seed_bootstrap_if_empty(db_path: str, enabled=True) -> None: rows=[] for i in range(180): lat=rng.uniform(-55,65); lon=rng.uniform(-170,170); mag=round(rng.uniform(1,6.4),1) - rows.append({'id':f'bootstrap-{i}','time_ms':now_ms-i*3600_000,'updated_ms':now_ms,'place':rng.choice(places),'magnitude':mag,'longitude':round(lon,3),'latitude':round(lat,3),'depth':round(rng.uniform(2,140),1),'tsunami':0,'significance':int(mag*100),'event_type':'earthquake','region':rough_region(lat,lon,rng.choice(places))}) + rows.append({'id':f'bootstrap-{i}','source':'bootstrap_demo','time_ms':now_ms-i*3600_000,'updated_ms':now_ms,'place':rng.choice(places),'magnitude':mag,'longitude':round(lon,3),'latitude':round(lat,3),'depth':round(rng.uniform(2,140),1),'tsunami':0,'significance':int(mag*100),'event_type':'earthquake','region':rough_region(lat,lon,rng.choice(places))}) upsert_earthquakes(db_path, rows, 'bootstrap_demo', True) def query_earthquakes(db_path: str, filters: dict[str,Any], limit: int, sort='time_desc') -> list[dict[str,Any]]: where=[]; args=[] - for key,col,op in [('mag_min','magnitude','>='),('mag_max','magnitude','<='),('lat_min','latitude','>='),('lat_max','latitude','<='),('lon_min','longitude','>='),('lon_max','longitude','<=')]: + for key,col,op in [('mag_min','magnitude','>='),('mag_max','magnitude','<='),('lat_min','latitude','>='),('lat_max','latitude','<=')]: if filters.get(key) is not None: where.append(f'{col} {op} ?'); args.append(filters[key]) + if filters.get('lon1') is not None and filters.get('lon2') is not None: + lon1=float(filters['lon1']); lon2=float(filters['lon2']) + if lon1 <= lon2: where.append('longitude BETWEEN ? AND ?'); args.extend([lon1,lon2]) + else: where.append('(longitude >= ? OR longitude <= ?)'); args.extend([lon1,lon2]) + else: + for key,col,op in [('lon_min','longitude','>='),('lon_max','longitude','<=')]: + if filters.get(key) is not None: where.append(f'{col} {op} ?'); args.append(filters[key]) if filters.get('start_time') is not None: where.append('time_ms >= ?'); args.append(filters['start_time']) if filters.get('end_time') is not None: where.append('time_ms <= ?'); args.append(filters['end_time']) + if filters.get('source') and filters.get('source')!='all': + src=filters['source']; src='usgs' if src in {'usgs','usgs_geojson'} else src + where.append('source = ?'); args.append(src) + if filters.get('source_feed') and filters.get('source_feed')!='all': where.append('source_feed = ?'); args.append(filters['source_feed']) + if filters.get('include_demo') is False: where.append('COALESCE(is_bootstrap,0)=0') sql='SELECT * FROM earthquakes '+(('WHERE '+ ' AND '.join(where)) if where else '') order={'time_asc':'time_ms ASC','magnitude_desc':'magnitude DESC','magnitude_asc':'magnitude ASC'}.get(sort,'time_ms DESC') sql += f' ORDER BY {order} LIMIT ?'; args.append(limit) diff --git a/app/services/sources/__init__.py b/app/services/sources/__init__.py new file mode 100644 index 0000000..92ce3aa --- /dev/null +++ b/app/services/sources/__init__.py @@ -0,0 +1 @@ +from .registry import SOURCE_REGISTRY, enabled_source_names, resolve_sources diff --git a/app/services/sources/auspass.py b/app/services/sources/auspass.py new file mode 100644 index 0000000..57b7cb2 --- /dev/null +++ b/app/services/sources/auspass.py @@ -0,0 +1,5 @@ +from .base import EarthquakeSource +class AusPassSource(EarthquakeSource): + def __init__(self): super().__init__('auspass','AusPass FDSN','Australia/Oceania','AusPass','https://auspass.edu.au/',False,True,['latest'],False) + def fetch(self, feed, params=None): return [] + def normalize(self, raw_event, feed): return None diff --git a/app/services/sources/base.py b/app/services/sources/base.py new file mode 100644 index 0000000..b230731 --- /dev/null +++ b/app/services/sources/base.py @@ -0,0 +1,25 @@ +from __future__ import annotations +from dataclasses import dataclass, field +from typing import Any +import requests + +@dataclass +class EarthquakeSource: + name: str + display_name: str + coverage: str + attribution: str + homepage_url: str + supports_realtime: bool = True + supports_historical: bool = False + supported_feeds: list[str] = field(default_factory=list) + enabled_by_default: bool = True + + def fetch(self, feed: str, params: dict[str, Any] | None = None) -> list[dict[str, Any]]: + raise NotImplementedError + def normalize(self, raw_event: dict[str, Any], feed: str) -> dict[str, Any] | None: + raise NotImplementedError + +def get_json(url: str, timeout: int=15, params: dict[str, Any] | None=None) -> Any: + r=requests.get(url, timeout=timeout, params=params, headers={'User-Agent':'earthquake-risk-dashboard/1.0'}) + r.raise_for_status(); return r.json() diff --git a/app/services/sources/bmkg.py b/app/services/sources/bmkg.py new file mode 100644 index 0000000..36ef832 --- /dev/null +++ b/app/services/sources/bmkg.py @@ -0,0 +1,24 @@ +from __future__ import annotations +from datetime import datetime, timezone +from .base import EarthquakeSource, get_json +from .normalizer import canonical_event, deterministic_id +class BMKGSource(EarthquakeSource): + def __init__(self): super().__init__('bmkg','BMKG Indonesia','Indonesia','BMKG Indonesia','https://www.bmkg.go.id/',True,False,['latest','felt','m5','tsunami']) + def fetch(self, feed, params=None): + url='https://data.bmkg.go.id/DataMKG/TEWS/autogempa.json' if feed in {'latest','tsunami'} else 'https://data.bmkg.go.id/DataMKG/TEWS/gempaterkini.json' + payload=get_json(url, int((params or {}).get('timeout',15))) + gempa=(payload.get('Infogempa') or {}).get('gempa', []) + if isinstance(gempa, dict): gempa=[gempa] + return [r for e in gempa if (r:=self.normalize(e, feed))] + def normalize(self, e, feed): + coords=str(e.get('Coordinates') or e.get('coordinates') or '').split(',') + lat=e.get('LintangDecimal') or (coords[0] if len(coords)>=2 else None); lon=e.get('BujurDecimal') or (coords[1] if len(coords)>=2 else None) + mag=e.get('Magnitude'); depth=str(e.get('Kedalaman') or '').split()[0] + time_s=' '.join(x for x in [e.get('Tanggal'), e.get('Jam')] if x) + time_ms=0 + for fmt in ('%d %b %Y %H:%M:%S %Z','%d-%b-%Y %H:%M:%S %Z'): + try: time_ms=int(datetime.strptime(time_s,fmt).replace(tzinfo=timezone.utc).timestamp()*1000); break + except Exception: pass + eid=e.get('EventID') or deterministic_id('bmkg',time_s,lat,lon,mag) + if not str(eid).startswith('bmkg:'): eid='bmkg:'+str(eid) + return canonical_event(event_id=eid, source='bmkg', source_feed=feed, time_ms=time_ms, place=e.get('Wilayah') or e.get('Dirasakan'), magnitude=mag, latitude=lat, longitude=lon, depth=depth, tsunami=1 if str(e.get('Potensi','')).lower().find('tsunami')>=0 else 0, url='https://www.bmkg.go.id/gempabumi/', raw_source=e) diff --git a/app/services/sources/emsc.py b/app/services/sources/emsc.py new file mode 100644 index 0000000..3c1e268 --- /dev/null +++ b/app/services/sources/emsc.py @@ -0,0 +1,3 @@ +from .usgs_fdsn import USGSFDSNSource +class EMSCSource(USGSFDSNSource): + def __init__(self): super().__init__(); self.name='emsc'; self.display_name='EMSC / SeismicPortal'; self.coverage='Europe/global'; self.attribution='EMSC / SeismicPortal (CC BY 4.0 where applicable)'; self.homepage_url='https://www.seismicportal.eu/'; self.supported_feeds=['latest','day','7day','30day'] diff --git a/app/services/sources/geofon.py b/app/services/sources/geofon.py new file mode 100644 index 0000000..056655c --- /dev/null +++ b/app/services/sources/geofon.py @@ -0,0 +1,5 @@ +from .base import EarthquakeSource +class GEOFONSource(EarthquakeSource): + def __init__(self): super().__init__('geofon','GFZ/GEOFON','Global/Europe','GFZ GEOFON','https://geofon.gfz-potsdam.de/',True,True,['latest','day','7day','30day']) + def fetch(self, feed, params=None): return [] + def normalize(self, raw_event, feed): return None diff --git a/app/services/sources/geonet.py b/app/services/sources/geonet.py new file mode 100644 index 0000000..8df85a8 --- /dev/null +++ b/app/services/sources/geonet.py @@ -0,0 +1,14 @@ +from __future__ import annotations +from typing import Any +from .base import EarthquakeSource, get_json +from .normalizer import canonical_event +class GeoNetSource(EarthquakeSource): + def __init__(self): super().__init__('geonet','GeoNet New Zealand','New Zealand/Oceania','GeoNet','https://www.geonet.org.nz/',True,True,['latest']) + def fetch(self, feed, params=None): + payload=get_json('https://api.geonet.org.nz/quake', int((params or {}).get('timeout',15))) + return [r for f in payload.get('features',[]) if (r:=self.normalize(f, feed))] + def normalize(self, f, feed): + p=f.get('properties') or {}; c=(f.get('geometry') or {}).get('coordinates') or [] + if len(c)<2: return None + eid=f.get('id') or p.get('publicID') or p.get('id') + return canonical_event(event_id='geonet:'+str(eid), source='geonet', source_feed=feed, time_ms=p.get('time') or p.get('origintime'), updated_ms=p.get('modificationtime'), place=p.get('locality'), magnitude=p.get('magnitude'), longitude=c[0], latitude=c[1], depth=c[2] if len(c)>2 else p.get('depth'), url='https://www.geonet.org.nz/earthquake/'+str(eid), detail_url=None, raw_source=f) diff --git a/app/services/sources/isc.py b/app/services/sources/isc.py new file mode 100644 index 0000000..487465d --- /dev/null +++ b/app/services/sources/isc.py @@ -0,0 +1,5 @@ +from .base import EarthquakeSource +class ISCSource(EarthquakeSource): + def __init__(self): super().__init__('isc','ISC FDSN','Global reviewed catalogue','International Seismological Centre','https://www.isc.ac.uk/',False,True,['latest','day','7day','30day']) + def fetch(self, feed, params=None): return [] + def normalize(self, raw_event, feed): return None diff --git a/app/services/sources/normalizer.py b/app/services/sources/normalizer.py new file mode 100644 index 0000000..8637694 --- /dev/null +++ b/app/services/sources/normalizer.py @@ -0,0 +1,47 @@ +from __future__ import annotations +import math, hashlib, json +from typing import Any +from app.utils import safe_float, rough_region + +def _finite(v): + try: return math.isfinite(float(v)) + except (TypeError, ValueError): return False + +def validate_latitude(lat: Any) -> bool: + return _finite(lat) and -90 <= float(lat) <= 90 + +def validate_longitude(lon: Any) -> bool: + return _finite(lon) and -180 <= float(lon) <= 180 + +def normalize_lat_lon(lat: Any, lon: Any) -> tuple[float, float] | None: + if not validate_latitude(lat) or not validate_longitude(lon): return None + return float(lat), float(lon) + +def validate_event_coordinates(event: dict[str, Any]) -> bool: + return validate_latitude(event.get('latitude')) and validate_longitude(event.get('longitude')) + +def deterministic_id(prefix: str, *parts: Any) -> str: + payload='|'.join(str(p) for p in parts) + return f"{prefix}:{hashlib.sha1(payload.encode('utf-8')).hexdigest()[:16]}" + +def canonical_event(*, event_id: str, source: str, source_feed: str, time_ms: Any, updated_ms: Any=None, + place: str='Unknown', magnitude: Any=None, latitude: Any=None, longitude: Any=None, + depth: Any=None, alert=None, tsunami: Any=0, significance=None, felt=None, url=None, + detail_url=None, event_type: str='earthquake', raw_source: dict[str, Any] | None=None, + **extra) -> dict[str, Any] | None: + latlon=normalize_lat_lon(latitude, longitude) + mag=safe_float(magnitude) + if latlon is None or mag is None: return None + lat, lon=latlon + dep=safe_float(depth) + row={ + 'id': event_id, 'source': source, 'source_feed': source_feed, + 'time_ms': int(time_ms or 0), 'updated_ms': int(updated_ms or time_ms or 0), + 'place': str(place or 'Unknown'), 'magnitude': float(mag), 'latitude': lat, 'longitude': lon, + 'depth': None if dep is None else float(dep), 'alert': alert, 'tsunami': int(tsunami or 0), + 'significance': significance, 'felt': felt, 'url': url, 'detail_url': detail_url, + 'event_type': event_type or 'earthquake', 'raw_source': raw_source or {}, + } + row.update(extra) + row['region']=rough_region(lat, lon, row['place']) + return row diff --git a/app/services/sources/registry.py b/app/services/sources/registry.py new file mode 100644 index 0000000..06558b6 --- /dev/null +++ b/app/services/sources/registry.py @@ -0,0 +1,20 @@ +from __future__ import annotations +from flask import current_app +from .usgs_geojson import USGSGeoJSONSource +from .usgs_fdsn import USGSFDSNSource +from .emsc import EMSCSource +from .isc import ISCSource +from .geofon import GEOFONSource +from .bmkg import BMKGSource +from .geonet import GeoNetSource +from .auspass import AusPassSource +SOURCE_REGISTRY={'usgs':USGSGeoJSONSource(),'usgs_geojson':USGSGeoJSONSource(),'usgs_fdsn':USGSFDSNSource(),'emsc':EMSCSource(),'isc':ISCSource(),'geofon':GEOFONSource(),'bmkg':BMKGSource(),'geonet':GeoNetSource(),'auspass':AusPassSource()} +def enabled_source_names(): + raw=current_app.config.get('ENABLED_SOURCES','usgs_geojson') + names=[s.strip() for s in raw.split(',') if s.strip()] + if not current_app.config.get('SOURCE_ENABLE_AUSPASS') and 'auspass' in names: names.remove('auspass') + return [n for n in names if n in SOURCE_REGISTRY] +def resolve_sources(requested: str): + if requested in ('all',''): return [SOURCE_REGISTRY[n] for n in enabled_source_names()] + if requested=='usgs': requested='usgs_geojson' + return [SOURCE_REGISTRY[requested]] if requested in SOURCE_REGISTRY else [SOURCE_REGISTRY['usgs_geojson']] diff --git a/app/services/sources/usgs_fdsn.py b/app/services/sources/usgs_fdsn.py new file mode 100644 index 0000000..9ce6e8d --- /dev/null +++ b/app/services/sources/usgs_fdsn.py @@ -0,0 +1,16 @@ +from __future__ import annotations +from typing import Any +from .base import EarthquakeSource, get_json +from .usgs_geojson import USGSGeoJSONSource +class USGSFDSNSource(EarthquakeSource): + def __init__(self): super().__init__('usgs_fdsn','USGS FDSN Event API','Global','USGS Earthquake Hazards Program','https://earthquake.usgs.gov/fdsnws/event/1/',True,True,['latest','day','7day','30day']) + def fetch(self, feed, params=None): + p=dict(params or {}); p.setdefault('format','geojson'); p.setdefault('limit', p.get('limit', 1000)); + p={k:v for k,v in p.items() if k in {'format','starttime','endtime','minmagnitude','maxmagnitude','mindepth','maxdepth','latitude','longitude','maxradiuskm','minlatitude','maxlatitude','minlongitude','maxlongitude','limit','orderby'} and v not in (None,'')} + payload=get_json('https://earthquake.usgs.gov/fdsnws/event/1/query', int((params or {}).get('timeout',15)), p) + parser=USGSGeoJSONSource(); rows=[] + for f in payload.get('features',[]): + row=parser.normalize(f, feed) + if row: row['id']=row['id'].replace('usgs:','usgs_fdsn:',1); row['source']='usgs_fdsn'; rows.append(row) + return rows + def normalize(self, raw_event, feed): return USGSGeoJSONSource().normalize(raw_event, feed) diff --git a/app/services/sources/usgs_geojson.py b/app/services/sources/usgs_geojson.py new file mode 100644 index 0000000..20085cf --- /dev/null +++ b/app/services/sources/usgs_geojson.py @@ -0,0 +1,14 @@ +from __future__ import annotations +from typing import Any +from .base import EarthquakeSource, get_json +from .normalizer import canonical_event +FEEDS={'hour':'all_hour.geojson','day':'all_day.geojson','7day':'all_week.geojson','30day':'all_month.geojson','significant':'significant_month.geojson','m1':'1.0_month.geojson','m2.5':'2.5_month.geojson','m4.5':'4.5_month.geojson'} +class USGSGeoJSONSource(EarthquakeSource): + def __init__(self): super().__init__('usgs_geojson','USGS GeoJSON','Global','USGS Earthquake Hazards Program','https://earthquake.usgs.gov/',True,False,list(FEEDS)) + def fetch(self, feed, params=None): + payload=get_json('https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/'+FEEDS.get(feed,FEEDS['day']), int((params or {}).get('timeout',15))) + return [self.normalize(f, feed) for f in payload.get('features',[]) if self.normalize(f, feed)] + def normalize(self, f: dict[str,Any], feed: str): + props=f.get('properties') or {}; coords=(f.get('geometry') or {}).get('coordinates') or [] + if len(coords)<2 or not f.get('id'): return None + return canonical_event(event_id='usgs:'+str(f.get('id')), source='usgs', source_feed=feed, time_ms=props.get('time'), updated_ms=props.get('updated'), place=props.get('place'), magnitude=props.get('mag'), longitude=coords[0], latitude=coords[1], depth=coords[2] if len(coords)>2 else None, alert=props.get('alert'), tsunami=props.get('tsunami'), significance=props.get('sig'), felt=props.get('felt'), url=props.get('url'), detail_url=props.get('detail'), event_type=props.get('type'), raw_source=f, network=props.get('net'), code=props.get('code'), mag_type=props.get('magType'), cdi=props.get('cdi'), mmi=props.get('mmi'), status=props.get('status'), gap=props.get('gap'), dmin=props.get('dmin'), rms=props.get('rms'), nst=props.get('nst')) diff --git a/app/services/usgs_client.py b/app/services/usgs_client.py index a448c6e..f35aa3a 100644 --- a/app/services/usgs_client.py +++ b/app/services/usgs_client.py @@ -16,7 +16,7 @@ def parse_geojson_features(payload: dict[str, Any]) -> list[dict[str, Any]]: if lon is None or lat is None or mag is None or not (-180<=lon<=180) or not (-90<=lat<=90): continue place=str(props.get('place') or 'Unknown') row={ - 'id': str(f.get('id')), 'time_ms': int(props.get('time') or 0), 'updated_ms': int(props.get('updated') or 0), + 'id': str(f.get('id')), 'source':'usgs', 'time_ms': int(props.get('time') or 0), 'updated_ms': int(props.get('updated') or 0), 'place': place, 'magnitude': float(mag), 'longitude': float(lon), 'latitude': float(lat), 'depth': float(depth or 0), 'alert': props.get('alert'), 'tsunami': int(props.get('tsunami') or 0), 'significance': props.get('sig'), 'felt': props.get('felt'), 'cdi': props.get('cdi'), 'mmi': props.get('mmi'), 'status': props.get('status'), 'event_type': props.get('type'), diff --git a/static/js/dashboard.js b/static/js/dashboard.js index ac0ebd4..929348c 100644 --- a/static/js/dashboard.js +++ b/static/js/dashboard.js @@ -1 +1 @@ -let charts=[], lastRows=[];const $=id=>document.getElementById(id);function chart(id,type,labels,data,label){const c=new Chart($(id),{type,data:{labels,datasets:[{label,data,backgroundColor:['#10b981','#f59e0b','#fb923c','#ef4444','#8b5cf6'],borderColor:'#7aa2ff'}]},options:{responsive:true}});charts.push(c)}function csv(rows){const cols=['id','time_ms','place','magnitude','latitude','longitude','depth','alert','tsunami','url'];return [cols.join(','),...rows.map(r=>cols.map(c=>`"${String(r[c]??'').replaceAll('"','""')}"`).join(','))].join('\n')}async function load(force=false){charts.forEach(c=>c.destroy());charts=[];const p=new URLSearchParams({feed:$('feed').value,mag_min:$('magMin').value,limit:$('limit').value,refresh:force?'1':'0'});const [eq,sum,fc]=await Promise.all([fetch('/api/earthquakes?'+p).then(r=>r.json()),fetch('/api/summary?'+p).then(r=>r.json()),fetch('/api/forecast?lat=37.7749&lon=-122.4194&radius_km=300').then(r=>r.json())]);lastRows=eq.data||[];const s=sum.summary;$('total').textContent=s.total_events;$('maxmag').textContent=s.max_magnitude??'-';$('avgmag').textContent=s.average_magnitude??'-';$('tsunami').textContent=s.tsunami_count;$('source').textContent=`Source ${eq.source}; count ${eq.count}; ${eq.warnings?.join(' ')||''}`;chart('trend','line',s.hourly_trend.map(x=>x.period).slice(-24),s.hourly_trend.map(x=>x.count).slice(-24),'Events');chart('magChart','doughnut',Object.keys(s.magnitude_distribution),Object.values(s.magnitude_distribution),'Magnitude');chart('depthChart','bar',Object.keys(s.depth_distribution),Object.values(s.depth_distribution),'Depth');$('regions').innerHTML=s.top_regions.map(x=>`
  • ${x[0]} ${x[1]}
    • `).join('');$('events').innerHTML=lastRows.filter(r=>(r.magnitude||0)>=4.5||r.tsunami).slice(0,10).map(r=>`M${r.magnitude}${r.place}${new Date(r.time_ms).toLocaleString()}`).join('')||'No significant events in current filter.';$('forecastCard').innerHTML=`
    ${fc.risk_score}
    ${fc.probability_bucket}

    ${fc.explanation}

    `} $('apply').onclick=()=>load(false);$('refresh').onclick=()=>load(true);$('export').onclick=()=>{const a=document.createElement('a');a.href=URL.createObjectURL(new Blob([csv(lastRows)],{type:'text/csv'}));a.download='earthquakes.csv';a.click()};load();setInterval(()=>load(false),120000); +let charts=[], lastRows=[];const $=id=>document.getElementById(id);function chart(id,type,labels,data,label){const c=new Chart($(id),{type,data:{labels,datasets:[{label,data,backgroundColor:['#10b981','#f59e0b','#fb923c','#ef4444','#8b5cf6'],borderColor:'#7aa2ff'}]},options:{responsive:true}});charts.push(c)}function csv(rows){const cols=['id','time_ms','place','magnitude','latitude','longitude','depth','alert','tsunami','url'];return [cols.join(','),...rows.map(r=>cols.map(c=>`"${String(r[c]??'').replaceAll('"','""')}"`).join(','))].join('\n')}async function load(force=false){charts.forEach(c=>c.destroy());charts=[];const p=new URLSearchParams({source:$('source').value,feed:$('feed').value,mag_min:$('magMin').value,limit:$('limit').value,refresh:force?'1':'0'});const [eq,sum,fc]=await Promise.all([fetch('/api/earthquakes?'+p).then(r=>r.json()),fetch('/api/summary?'+p).then(r=>r.json()),fetch('/api/forecast?lat=37.7749&lon=-122.4194&radius_km=300').then(r=>r.json())]);lastRows=eq.data||[];const s=sum.summary;$('total').textContent=s.total_events;$('maxmag').textContent=s.max_magnitude??'-';$('avgmag').textContent=s.average_magnitude??'-';$('tsunami').textContent=s.tsunami_count;$('source').textContent=`Source ${eq.meta?.source||eq.source}; feed ${eq.meta?.feed}; status ${eq.meta?.refresh_result}; count ${eq.count}; ${eq.warnings?.join(' ')||''}`;chart('trend','line',s.hourly_trend.map(x=>x.period).slice(-24),s.hourly_trend.map(x=>x.count).slice(-24),'Events');chart('magChart','doughnut',Object.keys(s.magnitude_distribution),Object.values(s.magnitude_distribution),'Magnitude');chart('depthChart','bar',Object.keys(s.depth_distribution),Object.values(s.depth_distribution),'Depth');$('regions').innerHTML=s.top_regions.map(x=>`
  • ${x[0]} ${x[1]}
    • `).join('');$('events').innerHTML=lastRows.filter(r=>(r.magnitude||0)>=4.5||r.tsunami).slice(0,10).map(r=>`M${r.magnitude}${r.place}${new Date(r.time_ms).toLocaleString()}`).join('')||'No significant events in current filter.';$('forecastCard').innerHTML=`
    ${fc.risk_score}
    ${fc.probability_bucket}

    ${fc.explanation}

    `} $('apply').onclick=()=>load(false);$('refresh').onclick=()=>load(true);$('export').onclick=()=>{const a=document.createElement('a');a.href=URL.createObjectURL(new Blob([csv(lastRows)],{type:'text/csv'}));a.download='earthquakes.csv';a.click()};load();setInterval(()=>load(false),120000); diff --git a/static/js/forecast.js b/static/js/forecast.js index 944582b..e898f20 100644 --- a/static/js/forecast.js +++ b/static/js/forecast.js @@ -1 +1 @@ -function preset(a,b){lat.value=a;lon.value=b;run()}async function run(){const p=new URLSearchParams({lat:lat.value,lon:lon.value,radius_km:radius.value,hours:hours.value,min_magnitude:minmag.value});const r=await fetch('/api/forecast?'+p).then(x=>x.json());score.textContent=r.risk_score;score.className='score '+r.probability_bucket;bucket.textContent=`${r.probability_bucket} · confidence ${r.model_confidence}`;explain.textContent=r.explanation;warn.textContent=r.warning;features.innerHTML=r.top_contributing_features.map(f=>`
  • ${f.feature}: ${f.value}
    • `).join('');nearby.innerHTML=r.nearby_recent_events.map(q=>`M${q.magnitude}${q.place}${q.distance_km} km`).join('')||'No nearby cached events.'}document.getElementById('run').onclick=run;run(); +function preset(a,b){lat.value=a;lon.value=b;run()}async function run(){const p=new URLSearchParams({source:source.value,feed:feed.value,lat:lat.value,lon:lon.value,radius_km:radius.value,hours:hours.value,min_magnitude:minmag.value});const r=await fetch('/api/forecast?'+p).then(x=>x.json());score.textContent=r.risk_score;score.className='score '+r.probability_bucket;bucket.textContent=`${r.probability_bucket} · confidence ${r.model_confidence}`;explain.textContent=r.explanation;warn.textContent=r.warning;features.innerHTML=r.top_contributing_features.map(f=>`
  • ${f.feature}: ${f.value}
    • `).join('');nearby.innerHTML=r.nearby_recent_events.map(q=>`M${q.magnitude}${q.place}${q.distance_km} km`).join('')||'No nearby cached events.'}document.getElementById('run').onclick=run;run(); diff --git a/static/js/globe.js b/static/js/globe.js index 1982502..14799fc 100644 --- a/static/js/globe.js +++ b/static/js/globe.js @@ -3,6 +3,8 @@ const globeContainer = document.getElementById('globeViz'); const statusBanner = document.getElementById('statusBanner'); const stats = document.getElementById('stats'); + const sourceInput = document.getElementById('source'); + const feedInput = document.getElementById('feed'); const magMinInput = document.getElementById('magMin'); const autoRefreshInput = document.getElementById('autoRefresh'); const autoRotateInput = document.getElementById('autoRotate'); @@ -53,8 +55,9 @@ function toPoint(quake, colorMode, heightScale) { const magnitude = Number.isFinite(quake.magnitude) ? quake.magnitude : 0; - const lat = Number.isFinite(quake.latitude) ? quake.latitude : 0; - const lng = Number.isFinite(quake.longitude) ? quake.longitude : 0; + const lat = Number(quake.latitude); + const lng = Number(quake.longitude); + if (!Number.isFinite(lat) || !Number.isFinite(lng) || lat < -90 || lat > 90 || lng < -180 || lng > 180) return null; const depth = Number.isFinite(quake.depth) ? quake.depth : 0; const place = quake.place || 'Unknown location'; const timeMs = Number.isFinite(quake.time_ms) ? quake.time_ms : 0; @@ -69,8 +72,8 @@ timeMs, radius: 0.05 + Math.max(magnitude, 0) * 0.045, altitude: (0.01 + Math.max(magnitude, 0) * 0.025) * heightScale, - color: colorMode === 'depth' ? depthColor(depth) : magnitudeColor(magnitude), - label: `
    ${place}
    Magnitude: ${magnitude.toFixed(1)}
    Depth: ${depth.toFixed(1)} km
    Time: ${time}
    ` + color: quake.is_bootstrap ? '#8b5cf6' : (colorMode === 'depth' ? depthColor(depth) : magnitudeColor(magnitude)), + label: `
    ${place}
    Magnitude: ${magnitude.toFixed(1)}
    Depth: ${depth.toFixed(1)} km
    Time: ${time}
    Source: ${quake.is_bootstrap?'Bootstrap Demo':(quake.source||'unknown')}
    Feed: ${quake.source_feed||'n/a'}
    ` }; } @@ -87,6 +90,8 @@ const heightScale = parseFloat(heightScaleInput.value) || 1; const params = new URLSearchParams({ + source: sourceInput.value, + feed: feedInput.value, mag_min: Number.isFinite(magMin) ? String(magMin) : '0', mag_max: '10', refresh: forceRefresh ? '1' : '0' @@ -104,21 +109,24 @@ const rows = Array.isArray(payload.data) ? payload.data : []; const filteredRows = rows.filter((row) => isRecent(row.time_ms, recentHours)); const cappedRows = Number.isFinite(maxPoints) ? filteredRows.slice(0, maxPoints) : filteredRows; - const points = cappedRows.map((row) => toPoint(row, colorMode, heightScale)); + const mapped = cappedRows.map((row) => toPoint(row, colorMode, heightScale)); + const points = mapped.filter(Boolean); + const skipped = mapped.length - points.length; world.pointsData(points); if (points.length === 0) { updateStatus('No earthquakes match your current filters.'); } else { - updateStatus('Data loaded. Hover points for details; click a point to focus.'); + const warnings = payload.meta?.warnings || []; + updateStatus((warnings.length ? warnings.join(' ') : 'Data loaded. Hover points for details; click a point to focus.') + (skipped ? ` Skipped ${skipped} invalid coordinate records.` : '')); } const lastRefresh = payload.meta?.last_refresh_epoch ? new Date(payload.meta.last_refresh_epoch * 1000).toLocaleString() : 'n/a'; - const source = payload.meta?.source || 'unknown'; - stats.textContent = `Displayed: ${points.length}/${rows.length} | Source: ${source} | Last updated: ${lastRefresh}`; + const source = payload.meta?.source || 'unknown'; const feed = payload.meta?.feed || 'n/a'; const status = payload.meta?.refresh_result || 'cache'; + stats.textContent = `Displayed: ${points.length}/${rows.length} | Source: ${source} | Feed: ${feed} | Status: ${status} | Last updated: ${lastRefresh}`; } catch (err) { world.pointsData([]); stats.textContent = 'Displayed: 0'; @@ -151,7 +159,7 @@ } refreshButton.addEventListener('click', () => fetchEarthquakes(true)); - [magMinInput, recentHoursInput, colorModeInput, maxPointsInput, heightScaleInput] + [sourceInput, feedInput, magMinInput, recentHoursInput, colorModeInput, maxPointsInput, heightScaleInput] .forEach((el) => el.addEventListener('change', () => fetchEarthquakes(false))); autoRefreshInput.addEventListener('change', () => setAutoRefresh(autoRefreshInput.checked)); autoRotateInput.addEventListener('change', () => setAutoRotate(autoRotateInput.checked)); diff --git a/static/js/map.js b/static/js/map.js index 6361b14..6413ca8 100644 --- a/static/js/map.js +++ b/static/js/map.js @@ -1 +1 @@ -const map=L.map('map').setView([20,0],2);L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',{attribution:'© OpenStreetMap'}).addTo(map);let layer=L.layerGroup().addTo(map);function color(m){return m>=6?'#ef4444':m>=4?'#f97316':m>=2?'#f59e0b':'#10b981'}async function load(useBounds=false){layer.clearLayers();const p=new URLSearchParams({feed:feed.value,mag_min:mag.value,limit:1500});if(useBounds){const b=map.getBounds();p.set('lat1',b.getSouth());p.set('lat2',b.getNorth());p.set('lon1',b.getWest());p.set('lon2',b.getEast())}const payload=await fetch('/api/earthquakes?'+p).then(r=>r.json());(payload.data||[]).forEach(q=>{L.circleMarker([q.latitude,q.longitude],{radius:4+(q.magnitude||0)*2,color:color(q.magnitude||0),fillOpacity:.65}).bindPopup(`${q.place}
    M ${q.magnitude} · ${q.depth} km
    ${new Date(q.time_ms).toLocaleString()}
    ${q.alert?`Alert: ${q.alert}
    `:''}USGS detail`).addTo(layer)});status.textContent=`${payload.count} events · ${payload.source}`}document.getElementById('load').onclick=()=>load(false);document.getElementById('bounds').onclick=()=>load(true);load(); +const map=L.map('map').setView([20,0],2);L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',{attribution:'© OpenStreetMap'}).addTo(map);let layer=L.layerGroup().addTo(map);function color(m){return m>=6?'#ef4444':m>=4?'#f97316':m>=2?'#f59e0b':'#10b981'}function sourceLabel(q){return q.is_bootstrap?'Bootstrap Demo':({usgs:'USGS',usgs_fdsn:'USGS FDSN',emsc:'EMSC',geonet:'GeoNet',bmkg:'BMKG',isc:'ISC',geofon:'GEOFON',auspass:'AusPass'}[q.source]||q.source||'Unknown')}async function load(useBounds=false){layer.clearLayers();const p=new URLSearchParams({source:source.value,feed:feed.value,mag_min:mag.value,limit:1500,include_demo:'false'});if(useBounds){const b=map.getBounds();p.set('lat1',b.getSouth());p.set('lat2',b.getNorth());p.set('lon1',b.getWest());p.set('lon2',b.getEast())}const payload=await fetch('/api/earthquakes?'+p).then(r=>r.json());(payload.data||[]).filter(q=>Number.isFinite(q.latitude)&&Number.isFinite(q.longitude)).forEach(q=>{const demo=!!q.is_bootstrap;L.circleMarker([q.latitude,q.longitude],{radius:4+(q.magnitude||0)*2,color:demo?'#8b5cf6':color(q.magnitude||0),fillColor:demo?'#9ca3af':color(q.magnitude||0),dashArray:demo?'4 4':null,fillOpacity:.65,weight:demo?3:1}).bindPopup(`${q.place}
    M ${q.magnitude} · ${q.depth??'n/a'} km
    ${new Date(q.time_ms).toLocaleString()}
    Source: ${sourceLabel(q)}
    Feed: ${q.source_feed||payload.meta?.feed||'n/a'}
    ${q.alert?`Alert: ${q.alert}
    `:''}${q.tsunami?`Tsunami flag: ${q.tsunami}
    `:''}${q.url?`Source detail`:''}`).addTo(layer)});const m=payload.meta||{};demoBanner.style.display=m.demo_count?'block':'none';status.textContent=`${m.count||payload.count} events · source ${m.source} · feed ${m.feed} · ${m.refresh_result} · ${(m.warnings||[]).join(' ')}`}document.getElementById('load').onclick=()=>load(false);document.getElementById('bounds').onclick=()=>load(true);load(); diff --git a/templates/dashboard.html b/templates/dashboard.html index 53e7c35..498fdd7 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -1 +1 @@ -{% extends 'base.html' %}{% block title %}Earthquake Analytics Dashboard{% endblock %}{% block head %}{% endblock %}{% block content %}

    Earthquake Monitoring Dashboard

    Live USGS feeds with resilient SQLite cache, analytics, and honest probabilistic risk scoring.

    Open Map Experimental Forecast
    Total events
    0
    Max magnitude
    -
    Average magnitude
    -
    Tsunami flags
    0

    Hourly trend

    Magnitude distribution

    Depth distribution

    Experimental model forecast

    Loading...

    Not deterministic earthquake prediction.

    Top regions

    Significant events

    {% endblock %}{% block scripts %}{% endblock %} +{% extends 'base.html' %}{% block title %}Earthquake Analytics Dashboard{% endblock %}{% block head %}{% endblock %}{% block content %}

    Earthquake Monitoring Dashboard

    Live USGS feeds with resilient SQLite cache, analytics, and honest probabilistic risk scoring.

    Open Map Experimental Forecast
    Total events
    0
    Max magnitude
    -
    Average magnitude
    -
    Tsunami flags
    0

    Hourly trend

    Magnitude distribution

    Depth distribution

    Experimental model forecast

    Loading...

    Not deterministic earthquake prediction.

    Top regions

    Significant events

    {% endblock %}{% block scripts %}{% endblock %} diff --git a/templates/forecast.html b/templates/forecast.html index 4e82169..32c2c88 100644 --- a/templates/forecast.html +++ b/templates/forecast.html @@ -1 +1 @@ -{% extends 'base.html' %}{% block title %}Probabilistic Forecast{% endblock %}{% block head %}{% endblock %}{% block content %}

    Experimental Probabilistic Forecast

    Regional risk scoring and seismic activity trend analysis. This is not deterministic earthquake prediction.

    Risk score

    --

    Contributing features

    Nearby recent events

    {% endblock %}{% block scripts %}{% endblock %} +{% extends 'base.html' %}{% block title %}Probabilistic Forecast{% endblock %}{% block head %}{% endblock %}{% block content %}

    Experimental Probabilistic Forecast

    Regional risk scoring and seismic activity trend analysis. This is not deterministic earthquake prediction.

    Risk score

    --

    Contributing features

    Nearby recent events

    {% endblock %}{% block scripts %}{% endblock %} diff --git a/templates/globe.html b/templates/globe.html index 29d8d4d..924e881 100644 --- a/templates/globe.html +++ b/templates/globe.html @@ -19,7 +19,7 @@

    Live Earthquake 3D Globe

    -
    +
    @@ -72,7 +72,7 @@

    Live Earthquake 3D Globe

    6+
    -
    Loading...
    +
    Loading...
    Demo/bootstrapPopup includes source/feed/status
    diff --git a/templates/map.html b/templates/map.html index 0230e14..f189015 100644 --- a/templates/map.html +++ b/templates/map.html @@ -1 +1 @@ -{% extends 'base.html' %}{% block title %}Earthquake Map{% endblock %}{% block head %}{% endblock %}{% block content %}

    2D Earthquake Map

    Markers are sized and colored by magnitude. Use the viewport button to query map bounds.

    {% endblock %}{% block scripts %}{% endblock %} +{% extends 'base.html' %}{% block title %}Earthquake Map{% endblock %}{% block head %}{% endblock %}{% block content %}

    2D Earthquake Map

    Markers are sized and colored by magnitude. Use the viewport button to query map bounds.

    Legends: color=magnitude; dashed purple/gray=demo data; popup shows source/feed/status.
    {% endblock %}{% block scripts %}{% endblock %} diff --git a/tests/test_app.py b/tests/test_app.py index 36217e5..caf5156 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -136,3 +136,42 @@ def fake_fetch(url, timeout): assert first.json['meta']['source'] == feed assert len(calls) == len(feeds) + +def test_normalizers_and_source_adapters_reject_bad_coordinates(): + from app.services.sources.normalizer import validate_latitude, validate_longitude, validate_event_coordinates + from app.services.sources.usgs_geojson import USGSGeoJSONSource + from app.services.sources.bmkg import BMKGSource + from app.services.sources.geonet import GeoNetSource + import math + assert not validate_latitude(math.nan) + assert not validate_longitude(math.inf) + assert not validate_event_coordinates({'latitude': 130, 'longitude': 45}) + usgs=USGSGeoJSONSource().normalize(SAMPLE['features'][0], 'day') + assert usgs['latitude'] == 37.2 and usgs['longitude'] == -122.1 and usgs['id'].startswith('usgs:') + assert USGSGeoJSONSource().normalize({'id':'bad','properties':{'mag':1},'geometry':{'coordinates':[130,95,1]}}, 'day') is None + bmkg=BMKGSource().normalize({'Coordinates':'-6.12,130.42','Magnitude':'4.8','Kedalaman':'42 km','Wilayah':'Indonesia'}, 'latest') + assert bmkg['latitude'] == -6.12 and bmkg['longitude'] == 130.42 + geonet=GeoNetSource().normalize({'id':'nz1','properties':{'magnitude':3,'locality':'NZ','time':1700000000000},'geometry':{'coordinates':[174.7,-41.2,12]}}, 'latest') + assert geonet['latitude'] == -41.2 and geonet['longitude'] == 174.7 + +def test_source_feed_demo_and_antimeridian_filters(tmp_path): + from app.services.cache import init_db, upsert_earthquakes, query_earthquakes + db=str(tmp_path/'filters.db'); init_db(db) + upsert_earthquakes(db,[{'id':'usgs:day','source':'usgs','time_ms':1,'updated_ms':1,'place':'A','magnitude':2,'longitude':175,'latitude':10,'depth':1}], 'day') + upsert_earthquakes(db,[{'id':'usgs:month','source':'usgs','time_ms':2,'updated_ms':2,'place':'B','magnitude':3,'longitude':-175,'latitude':-10,'depth':1}], '30day') + upsert_earthquakes(db,[{'id':'emsc:x','source':'emsc','time_ms':3,'updated_ms':3,'place':'C','magnitude':4,'longitude':0,'latitude':0,'depth':1}], 'day') + upsert_earthquakes(db,[{'id':'bootstrap-x','source':'bootstrap_demo','time_ms':4,'updated_ms':4,'place':'Demo','magnitude':5,'longitude':50,'latitude':5,'depth':1}], 'bootstrap_demo', True) + assert [r['id'] for r in query_earthquakes(db, {'source':'usgs','source_feed':'day','include_demo':False}, 10)] == ['usgs:day'] + assert [r['id'] for r in query_earthquakes(db, {'source':'usgs','source_feed':'30day','include_demo':False}, 10)] == ['usgs:month'] + assert all(not r['is_bootstrap'] for r in query_earthquakes(db, {'source':'all','include_demo':False}, 10)) + assert any(r['is_bootstrap'] for r in query_earthquakes(db, {'source':'all','include_demo':True}, 10)) + ids={r['id'] for r in query_earthquakes(db, {'lon1':170,'lon2':-170,'include_demo':False}, 10)} + assert {'usgs:day','usgs:month'} <= ids and 'emsc:x' not in ids + assert [r['id'] for r in query_earthquakes(db, {'lon1':-10,'lon2':10,'include_demo':False}, 10)] == ['emsc:x'] + +def test_sources_and_filtered_api(client): + assert client.get('/api/sources').status_code == 200 + assert client.get('/api/regions?source=usgs&feed=day').status_code == 200 + assert client.get('/api/timeseries?source=usgs&feed=day').status_code == 200 + r=client.get('/api/earthquakes?source=usgs&feed=day&include_demo=false') + assert r.status_code == 200 and r.json['meta']['demo_count'] == 0