diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b5adcfc --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +USGS_FEED_URL=https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.geojson +CACHE_TTL_SECONDS=300 +SQLITE_DB_PATH=data/earthquakes.db +DEFAULT_FEED_WINDOW=day +MODEL_PATH=models/earthquake_risk.joblib +ENABLE_BOOTSTRAP_DATA=true +ENABLE_MODEL_TRAINING_ON_STARTUP=false +MAX_API_LIMIT=2000 +ADMIN_TOKEN=change-me-for-admin-training diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..9f26c24 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,12 @@ +name: tests +on: [push, pull_request] +jobs: + pytest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: {python-version: '3.11'} + - run: pip install -r requirements.txt + - run: pytest + - run: python -m compileall . diff --git a/.gitignore b/.gitignore index 4a98059..148587c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,10 @@ __pycache__/ -*.pyc -data/ +*.py[cod] +.pytest_cache/ +.env .venv/ +venv/ +data/*.db* +models/*.joblib +models/*.json +.DS_Store diff --git a/.python-version b/.python-version new file mode 100644 index 0000000..ed7d51a --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11.15 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..a23c454 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +## 2026-06-26 + +- Refactored the Flask app into an application factory and service modules. +- Added robust SQLite cache schema, indexes, upserts, refresh metadata, and fallback handling. +- Added summary, regions, timeseries, forecast, model status/train, and deep health APIs. +- Added probabilistic risk scoring with heuristic fallback and optional scikit-learn training. +- Rebuilt dashboard, added Leaflet map, forecast page, and about page. +- Updated Render deployment, dependencies, environment documentation, tests, and CI. diff --git a/README.md b/README.md index ed37579..6bf806e 100644 --- a/README.md +++ b/README.md @@ -1,149 +1,99 @@ -# Live Earthquake Dashboard (Flask + JavaScript) +# QuakeRisk: Earthquake Monitoring & Probabilistic Forecasting Dashboard -Deployable Flask app for **live earthquake visualization** on Render, with resilient caching and two UI views: -- Chart dashboard (`/dashboard`) -- Interactive 3D globe (`/globe`) powered by `globe.gl` +A production-ready Flask application for monitoring USGS GeoJSON earthquake feeds, caching records in SQLite, visualizing activity with Chart.js/Leaflet/globe.gl, and providing **experimental probabilistic earthquake forecasting**. -## Features +> Scientific disclaimer: earthquakes cannot be predicted deterministically with reliable exact time, location, and magnitude. This app only provides regional risk scoring, magnitude likelihood estimation, aftershock/seismic activity trend analysis, and educational uncertainty warnings. -- Uses the **USGS live GeoJSON feed** (`all_day`) as primary data source. -- Caches latest feed data in SQLite (`data/earthquakes.db`) for reliability when the external API is temporarily unavailable. -- Exposes `GET /api/earthquakes` with magnitude/geo filters and optional force refresh. -- Includes: - - chart-based dashboard for quick analytics - - 3D globe visualization with tooltip details, magnitude sizing/coloring, and auto-refresh controls -- Ready for Render deployment with Gunicorn via `Procfile`. +## Screenshots + +Add screenshots after deployment for the dashboard, 2D map, 3D globe, and forecast page. ## Architecture -- **Backend:** Flask (`main.py`) -- **Data Source:** USGS Earthquake Feed -- **Cache:** SQLite local cache (auto-created) -- **Frontend:** - - `templates/dashboard.html` (Chart.js dashboard) - - `templates/globe.html` + `static/js/globe.js` + `static/css/globe.css` (`globe.gl` view) +- `main.py` keeps `gunicorn main:app` compatibility. +- `app/__init__.py` creates the Flask app. +- `app/routes.py` exposes HTML pages and JSON APIs. +- `app/services/usgs_client.py` fetches/parses USGS GeoJSON. +- `app/services/cache.py` manages SQLite tables, indexes, upserts, and refresh metadata. +- `app/services/analytics.py` powers dashboard summary aggregations. +- `app/services/model.py` implements a heuristic baseline plus optional scikit-learn classifier. +- `templates/` and `static/` contain responsive frontend pages. +- `tests/` contains pytest coverage for parsing, cache, APIs, and forecast fallback. ## Routes -- `/` and `/dashboard`: existing dashboard view -- `/globe`: new 3D globe visualization page -- `/api/earthquakes`: JSON data endpoint used by both views -- `/health`: health check endpoint - -## API - -### `GET /api/earthquakes` - -Query params: -- `mag_min`, `mag_max` -- `lat1`, `lat2` -- `lon1`, `lon2` -- `refresh=1` (optional force live refresh) - -Response shape: - -```json -{ - "meta": { - "source": "live|cache|fallback_cache|bootstrap_cache", - "refresh_result": "live|cache|fallback_cache|bootstrap_cache", - "last_refresh_epoch": 0, - "count": 0, - "filters": {} - }, - "data": [ - { - "id": "...", - "time_ms": 0, - "place": "...", - "magnitude": 0, - "latitude": 0, - "longitude": 0, - "depth": 0 - } - ] -} +HTML: `/dashboard`, `/map`, `/globe`, `/forecast`, `/about`. + +APIs: `/api/earthquakes`, `/api/summary`, `/api/regions`, `/api/timeseries`, `/api/forecast`, `/api/model/status`, `POST /api/model/train`, `/health`, `/api/health/deep`. + +## API examples + +```bash +curl 'http://localhost:5000/api/earthquakes?feed=day&mag_min=2.5&limit=100' +curl 'http://localhost:5000/api/summary?feed=7day' +curl 'http://localhost:5000/api/forecast?lat=37.7749&lon=-122.4194&radius_km=300&hours=72&min_magnitude=2.5' ``` -## Local run +## ML forecast explanation + +The forecast endpoint returns a 0-100 risk score, bucket, expected activity count, model confidence, top contributing features, nearby events, and a warning. It always works through a baseline heuristic; if enough cached data exists, `POST /api/model/train` can train a scikit-learn classifier saved with joblib. + +Training is time-aware and reports accuracy, precision, recall, F1, confusion matrix, and ROC AUC when valid. It avoids deterministic claims and should not be used for emergency decisions. + +## Local setup ```bash -python3 -m venv .venv +python -m venv .venv source .venv/bin/activate pip install -r requirements.txt -python3 main.py +python main.py +``` + +Open . + +## Render deployment + +The included `Procfile` and `render.yaml` use: + +```bash +gunicorn main:app --bind 0.0.0.0:$PORT ``` -Open: -- Dashboard: `http://localhost:5000/dashboard` -- 3D globe: `http://localhost:5000/globe` - -## Globe view behavior - -The globe view calls `/api/earthquakes` directly and maps data as follows: -- `lat` ← `latitude` -- `lng` ← `longitude` -- point radius/altitude scale with `magnitude` -- point color reflects magnitude severity bands -- tooltip includes place, magnitude, depth, and time - -UI controls include: -- minimum magnitude filter -- recent time window filter (last 24h/12h/6h/3h or all cached points) -- color mode toggle (magnitude or depth) -- maximum rendered points cap -- point height scaling slider -- auto-refresh toggle (2-minute interval) -- auto-rotate toggle -- manual refresh button -- magnitude color legend -- status, count, source, and last updated timestamp - -Interaction: -- hover a point for details -- click a point to focus the camera on that earthquake - -If data is empty or unavailable, the page shows friendly status messaging instead of crashing. - -## Deploy to Render - -This repository includes both: -- `Procfile` (classic process command) -- `render.yaml` (Render Blueprint / IaC) - -### Option A: Deploy via `render.yaml` (recommended) - -1. Push the repo to GitHub. -2. In Render, choose **New +** → **Blueprint**. -3. Select this repository and apply the blueprint. - -### Option B: Manual Web Service setup - -1. Push the repo to GitHub. -2. In Render, create a **Web Service** from the repo. -3. Runtime: Python 3. -4. Build command: - ```bash - pip install -r requirements.txt - ``` -5. Start command: - ```bash - gunicorn main:app --bind 0.0.0.0:$PORT - ``` -6. Optional environment variables: - - `CACHE_TTL_SECONDS=300` - - `USGS_FEED_URL=https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.geojson` - -## Health check - -`GET /health` returns: - -```json -{"status":"ok"} +Health check path: `/health`. + +SQLite works out of the box for Render free-tier compatibility, but local disk can be ephemeral. For durable history, attach a Render persistent disk and set `SQLITE_DB_PATH` to that mount, or migrate the cache layer to Postgres later. + +## Environment variables + +See `.env.example` for all supported settings: `USGS_FEED_URL`, `CACHE_TTL_SECONDS`, `SQLITE_DB_PATH`, `DEFAULT_FEED_WINDOW`, `MODEL_PATH`, `ENABLE_BOOTSTRAP_DATA`, `ENABLE_MODEL_TRAINING_ON_STARTUP`, `MAX_API_LIMIT`, and `ADMIN_TOKEN`. + +## Model training + +Set `ADMIN_TOKEN`, then call: + +```bash +curl -X POST -H "X-Admin-Token: $ADMIN_TOKEN" http://localhost:5000/api/model/train ``` -## Notes +If no token is configured, the training endpoint is disabled. + +## Tests + +```bash +pytest +python -m compileall . +``` + +## Troubleshooting + +- If USGS is unavailable, APIs fall back to cached or clearly labeled bootstrap/demo data. +- If CDN assets fail, the 2D map and dashboard still provide monitoring views. +- If model training reports insufficient data, continue using the heuristic forecast until more cache history exists. + +## Future improvements -- On startup, the app creates cache tables and attempts refresh. -- If USGS is unreachable, the app serves last cached records. -- This behavior is suitable for Render free/limited instances with occasional transient network failures. +- Persistent Postgres backend. +- Region-specific model calibration. +- Background scheduled ingestion. +- Authentication for admin operations. +- Screenshot automation for README assets. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..3e986b4 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,19 @@ +from __future__ import annotations +import logging +from flask import Flask +from app.config import Config +from app.routes import bp +from app.services.cache import init_db, seed_bootstrap_if_empty, all_records +from app.services.model import train + +def create_app(config_class=Config): + app=Flask(__name__, template_folder='../templates', static_folder='../static') + app.config.from_object(config_class) + logging.basicConfig(level=logging.INFO) + init_db(app.config['SQLITE_DB_PATH']) + seed_bootstrap_if_empty(app.config['SQLITE_DB_PATH'], app.config['ENABLE_BOOTSTRAP_DATA']) + if app.config['ENABLE_MODEL_TRAINING_ON_STARTUP']: + try: train(all_records(app.config['SQLITE_DB_PATH']), app.config['MODEL_PATH']) + except Exception: app.logger.exception('Startup model training failed') + app.register_blueprint(bp) + return app diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..ce1d492 --- /dev/null +++ b/app/config.py @@ -0,0 +1,29 @@ +import os +from pathlib import Path + +BASE_DIR = Path(__file__).resolve().parent.parent + +class Config: + SECRET_KEY = os.getenv('SECRET_KEY', 'dev-not-secret') + USGS_FEED_URL = os.getenv('USGS_FEED_URL', 'https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.geojson') + CACHE_TTL_SECONDS = int(os.getenv('CACHE_TTL_SECONDS', '300')) + SQLITE_DB_PATH = os.getenv('SQLITE_DB_PATH', str(BASE_DIR / 'data' / 'earthquakes.db')) + DEFAULT_FEED_WINDOW = os.getenv('DEFAULT_FEED_WINDOW', 'day') + MODEL_PATH = os.getenv('MODEL_PATH', str(BASE_DIR / 'models' / 'earthquake_risk.joblib')) + ENABLE_BOOTSTRAP_DATA = os.getenv('ENABLE_BOOTSTRAP_DATA', 'true').lower() in {'1','true','yes','on'} + ENABLE_MODEL_TRAINING_ON_STARTUP = os.getenv('ENABLE_MODEL_TRAINING_ON_STARTUP', 'false').lower() in {'1','true','yes','on'} + MAX_API_LIMIT = int(os.getenv('MAX_API_LIMIT', '2000')) + ADMIN_TOKEN = os.getenv('ADMIN_TOKEN', '') + FLASK_ENV = os.getenv('FLASK_ENV', os.getenv('ENV', 'production')) + REQUEST_TIMEOUT_SECONDS = int(os.getenv('REQUEST_TIMEOUT_SECONDS', '15')) + + FEEDS = { + 'hour': 'https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_hour.geojson', + 'day': 'https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.geojson', + '7day': 'https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_week.geojson', + '30day': 'https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_month.geojson', + 'significant': 'https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/significant_month.geojson', + 'm1': 'https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/1.0_month.geojson', + 'm2.5': 'https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/2.5_month.geojson', + 'm4.5': 'https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/4.5_month.geojson', + } diff --git a/app/routes.py b/app/routes.py new file mode 100644 index 0000000..264378a --- /dev/null +++ b/app/routes.py @@ -0,0 +1,75 @@ +from __future__ import annotations +import time +from flask import Blueprint, current_app, jsonify, render_template, request +from app.services import analytics, model +from app.services.cache import all_meta, all_records, connect, count_records, get_meta, query_earthquakes, seed_bootstrap_if_empty, set_meta, upsert_earthquakes +from app.services.usgs_client import fetch_feed +from app.utils import bool_arg, parse_time_ms, safe_float, safe_int +bp=Blueprint('main',__name__) + +def feed_url(name): + feeds=current_app.config['FEEDS']; return current_app.config['USGS_FEED_URL'] if name=='custom' else feeds.get(name, feeds.get(current_app.config['DEFAULT_FEED_WINDOW'], current_app.config['USGS_FEED_URL'])) +def refresh_if_needed(feed='day', force=False): + db=current_app.config['SQLITE_DB_PATH']; ttl=current_app.config['CACHE_TTL_SECONDS']; last=int(get_meta(db,'last_successful_refresh_epoch','0') or 0) + if not force and int(time.time())-last <= ttl: return 'cache' + try: + rows=fetch_feed(feed_url(feed), current_app.config['REQUEST_TIMEOUT_SECONDS']); n=upsert_earthquakes(db,rows,feed,False) + return f'live:{n}' + except Exception as exc: + current_app.logger.warning('USGS refresh failed: %s', exc) + with connect(db) as c: set_meta(c,'last_refresh_error',str(exc)[:300]); set_meta(c,'last_refresh_source','fallback_cache'); c.commit() + if count_records(db)==0: seed_bootstrap_if_empty(db, current_app.config['ENABLE_BOOTSTRAP_DATA']); return 'bootstrap_fallback' + return 'fallback_cache' +def filters_from_args(): + mag_min=safe_float(request.args.get('mag_min'),0.0); mag_max=safe_float(request.args.get('mag_max'),10.0) + lat1=safe_float(request.args.get('lat1'),-90.0); lat2=safe_float(request.args.get('lat2'),90.0); lon1=safe_float(request.args.get('lon1'),-180.0); lon2=safe_float(request.args.get('lon2'),180.0) + mag_min,mag_max=sorted((mag_min,mag_max)); lat1=max(-90,min(90,lat1)); lat2=max(-90,min(90,lat2)); lon1=max(-180,min(180,lon1)); lon2=max(-180,min(180,lon2)); lat_min,lat_max=sorted((lat1,lat2)); lon_min,lon_max=sorted((lon1,lon2)) + return {'mag_min':mag_min,'mag_max':mag_max,'lat_min':lat_min,'lat_max':lat_max,'lon_min':lon_min,'lon_max':lon_max,'start_time':parse_time_ms(request.args.get('start_time')),'end_time':parse_time_ms(request.args.get('end_time'))} +@bp.route('/') +@bp.route('/dashboard') +def dashboard(): return render_template('dashboard.html') +@bp.route('/globe') +def globe(): return render_template('globe.html') +@bp.route('/map') +def map_page(): return render_template('map.html') +@bp.route('/forecast') +def forecast_page(): return render_template('forecast.html') +@bp.route('/about') +def about(): return render_template('about.html') +@bp.route('/api/earthquakes') +def api_earthquakes(): + feed=request.args.get('feed', current_app.config['DEFAULT_FEED_WINDOW']); refresh=refresh_if_needed(feed,bool_arg('refresh')) + limit=safe_int(request.args.get('limit'),500,1,current_app.config['MAX_API_LIMIT']); sort=request.args.get('sort','time_desc') + filt=filters_from_args(); rows=query_earthquakes(current_app.config['SQLITE_DB_PATH'],filt,limit,sort); meta=all_meta(current_app.config['SQLITE_DB_PATH']) + warnings=[] + if 'fallback' in refresh or meta.get('last_refresh_source') in {'bootstrap_demo','fallback_cache'}: warnings.append('Using cached or bootstrap data because live USGS refresh was unavailable or skipped.') + return jsonify({'meta':{'source':meta.get('last_refresh_source','cache'),'refresh_result':refresh,'last_refresh_epoch':int(meta.get('last_successful_refresh_epoch','0') or 0),'count':len(rows),'available_feeds':list(current_app.config['FEEDS'].keys())},'filters':filt,'count':len(rows),'source':meta.get('last_refresh_source','cache'),'warnings':warnings,'data':rows}) +@bp.route('/api/summary') +def api_summary(): + refresh_if_needed(request.args.get('feed', current_app.config['DEFAULT_FEED_WINDOW']), False) + rows=query_earthquakes(current_app.config['SQLITE_DB_PATH'],filters_from_args(),current_app.config['MAX_API_LIMIT'],'time_desc') + return jsonify({'summary':analytics.summarize(rows),'count':len(rows),'source':all_meta(current_app.config['SQLITE_DB_PATH']).get('last_refresh_source','cache')}) +@bp.route('/api/regions') +def api_regions(): return jsonify({'regions':analytics.regions(query_earthquakes(current_app.config['SQLITE_DB_PATH'],filters_from_args(),current_app.config['MAX_API_LIMIT'],'time_desc'))}) +@bp.route('/api/timeseries') +def api_timeseries(): return jsonify({'interval':request.args.get('interval','hour'),'series':analytics.timeseries(query_earthquakes(current_app.config['SQLITE_DB_PATH'],filters_from_args(),current_app.config['MAX_API_LIMIT'],'time_desc'), request.args.get('interval','hour'))}) +@bp.route('/api/forecast') +def api_forecast(): + lat=safe_float(request.args.get('lat'),37.7749); lon=safe_float(request.args.get('lon'),-122.4194); radius=safe_float(request.args.get('radius_km'),300); hours=safe_int(request.args.get('hours'),72,1,24*30); minmag=safe_float(request.args.get('min_magnitude'),2.5) + rows=all_records(current_app.config['SQLITE_DB_PATH'], current_app.config['MAX_API_LIMIT']); return jsonify(model.predict(rows,current_app.config['MODEL_PATH'],lat,lon,radius,hours,minmag)) +@bp.route('/api/model/train', methods=['POST']) +def api_train(): + token=current_app.config.get('ADMIN_TOKEN') + if not token or request.headers.get('X-Admin-Token') != token: return jsonify({'error':'training endpoint disabled or unauthorized'}), 403 + return jsonify(model.train(all_records(current_app.config['SQLITE_DB_PATH'],20000), current_app.config['MODEL_PATH'])) +@bp.route('/api/model/status') +def api_model_status(): return jsonify(model.status(current_app.config['MODEL_PATH'])) +@bp.route('/health') +def health(): return jsonify({'status':'ok'}) +@bp.route('/api/health/deep') +def deep_health(): + db=current_app.config['SQLITE_DB_PATH']; meta=all_meta(db); ok=True + try: records=count_records(db) + except Exception: ok=False; records=0 + st=model.status(current_app.config['MODEL_PATH']) + return jsonify({'app':'ok','database_ok':ok,'cache_ok':records>=0,'model_ok':st['model_exists'],'last_refresh':meta.get('last_successful_refresh_epoch'),'record_count':records,'model':st}) diff --git a/app/services/analytics.py b/app/services/analytics.py new file mode 100644 index 0000000..5ec0a70 --- /dev/null +++ b/app/services/analytics.py @@ -0,0 +1,36 @@ +from __future__ import annotations +from collections import Counter, defaultdict +import statistics +from datetime import datetime, timezone + +def mag_bucket(m): + if m is None: return 'unknown' + if m<2: return '0-1.9' + if m<3: return '2-2.9' + if m<4: return '3-3.9' + if m<5: return '4-4.9' + return '5+' +def depth_bucket(d): + if d is None: return 'unknown' + if d<10: return '0-10 km' + if d<35: return '10-35 km' + if d<70: return '35-70 km' + if d<150: return '70-150 km' + return '150+ km' +def summarize(rows): + mags=[r['magnitude'] for r in rows if r.get('magnitude') is not None]; depths=[r['depth'] for r in rows if r.get('depth') is not None] + newest=max(rows, key=lambda r:r.get('time_ms') or 0) if rows else None + return {'total_events':len(rows),'max_magnitude':max(mags) if mags else None,'average_magnitude':round(sum(mags)/len(mags),3) if mags else None,'median_magnitude':statistics.median(mags) if mags else None,'deepest_event':max(rows,key=lambda r:r.get('depth') or -999) if rows else None,'shallowest_event':min(rows,key=lambda r:r.get('depth') or 999999) if rows else None,'newest_event':newest,'high_risk_count':sum(1 for r in rows if (r.get('magnitude') or 0)>=4.5),'tsunami_count':sum(1 for r in rows if r.get('tsunami')),'significant_event_count':sum(1 for r in rows if (r.get('significance') or 0)>=600 or (r.get('magnitude') or 0)>=5),'top_regions':Counter(r.get('region') or 'Unknown' for r in rows).most_common(10),'magnitude_distribution':Counter(mag_bucket(r.get('magnitude')) for r in rows),'depth_distribution':Counter(depth_bucket(r.get('depth')) for r in rows),'hourly_trend':timeseries(rows,'hour'),'daily_trend':timeseries(rows,'day')} +def timeseries(rows, interval='hour'): + b=defaultdict(lambda:{'count':0,'avg_magnitude':0,'max_magnitude':None}); sums=defaultdict(float) + for r in rows: + ms=r.get('time_ms') or 0; dt=datetime.fromtimestamp(ms/1000,tz=timezone.utc) + key=dt.strftime('%Y-%m-%dT%H:00Z') if interval=='hour' else dt.strftime('%Y-%m-%d') + m=r.get('magnitude') or 0; b[key]['count']+=1; sums[key]+=m; b[key]['max_magnitude']=m if b[key]['max_magnitude'] is None else max(b[key]['max_magnitude'],m) + return [{'period':k,'count':v['count'],'avg_magnitude':round(sums[k]/v['count'],3),'max_magnitude':v['max_magnitude']} for k,v in sorted(b.items())] +def regions(rows): + out=[] + for reg,c in Counter(r.get('region') or 'Unknown' for r in rows).most_common(): + rr=[r for r in rows if (r.get('region') or 'Unknown')==reg]; mags=[r.get('magnitude') or 0 for r in rr] + out.append({'region':reg,'count':c,'max_magnitude':max(mags) if mags else None,'avg_magnitude':round(sum(mags)/len(mags),2) if mags else None}) + return out diff --git a/app/services/cache.py b/app/services/cache.py new file mode 100644 index 0000000..40cf491 --- /dev/null +++ b/app/services/cache.py @@ -0,0 +1,77 @@ +from __future__ import annotations +import json, random, sqlite3, time +from pathlib import Path +from typing import Any +from app.utils import rough_region + +COLUMNS=['id','time_ms','updated_ms','place','magnitude','longitude','latitude','depth','alert','tsunami','significance','felt','cdi','mmi','status','event_type','network','code','url','detail_url','mag_type','gap','dmin','rms','nst','region'] + +def connect(db_path: str) -> sqlite3.Connection: + Path(db_path).parent.mkdir(parents=True, exist_ok=True) + conn=sqlite3.connect(db_path, timeout=30) + conn.row_factory=sqlite3.Row + conn.execute('PRAGMA journal_mode=WAL'); conn.execute('PRAGMA busy_timeout=5000') + return conn + +def init_db(db_path: str) -> None: + with connect(db_path) as c: + c.execute('''CREATE TABLE IF NOT EXISTS earthquakes ( + id TEXT PRIMARY KEY, time_ms INTEGER, updated_ms INTEGER, place TEXT, magnitude REAL, + longitude REAL, latitude REAL, depth REAL, alert TEXT, tsunami INTEGER, significance INTEGER, + felt INTEGER, cdi REAL, mmi REAL, status TEXT, event_type TEXT, network TEXT, code TEXT, + url TEXT, detail_url TEXT, mag_type TEXT, gap REAL, dmin REAL, rms REAL, nst INTEGER, + region TEXT, ingested_at INTEGER, source_feed TEXT, is_bootstrap INTEGER DEFAULT 0)''') + c.execute('''CREATE TABLE IF NOT EXISTS refresh_metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL)''') + for idx,col in [('idx_quakes_time','time_ms'),('idx_quakes_mag','magnitude'),('idx_quakes_lat','latitude'),('idx_quakes_lon','longitude'),('idx_quakes_region','region')]: + c.execute(f'CREATE INDEX IF NOT EXISTS {idx} ON earthquakes({col})') + c.commit() + +def set_meta(c, k, v): c.execute('INSERT INTO refresh_metadata(key,value) VALUES(?,?) ON CONFLICT(key) DO UPDATE SET value=excluded.value',(k,str(v))) +def get_meta(db_path: str, k: str, default: str='') -> str: + with connect(db_path) as c: + r=c.execute('SELECT value FROM refresh_metadata WHERE key=?',(k,)).fetchone(); return r['value'] if r else default + +def all_meta(db_path: str) -> dict[str,str]: + with connect(db_path) as c: return {r['key']:r['value'] for r in c.execute('SELECT key,value FROM refresh_metadata')} + +def upsert_earthquakes(db_path: str, rows: list[dict[str,Any]], source_feed='unknown', is_bootstrap=False) -> int: + if not rows: return 0 + now=int(time.time()) + vals=[] + for r in rows: + r=dict(r); r.setdefault('region', rough_region(r.get('latitude'), r.get('longitude'), r.get('place',''))) + vals.append(tuple(r.get(col) for col in COLUMNS)+(now,source_feed,1 if is_bootstrap else 0)) + ph=','.join(['?']*(len(COLUMNS)+3)) + updates=','.join([f'{c}=excluded.{c}' for c in COLUMNS[1:]]+['ingested_at=excluded.ingested_at','source_feed=excluded.source_feed','is_bootstrap=excluded.is_bootstrap']) + with connect(db_path) as c: + c.executemany(f'INSERT INTO earthquakes({",".join(COLUMNS)},ingested_at,source_feed,is_bootstrap) VALUES({ph}) ON CONFLICT(id) DO UPDATE SET {updates}', vals) + set_meta(c,'last_successful_refresh_epoch',now); set_meta(c,'last_refresh_source',source_feed); set_meta(c,'last_record_count',len(rows)); c.commit() + return len(rows) + +def seed_bootstrap_if_empty(db_path: str, enabled=True) -> None: + if not enabled: return + with connect(db_path) as c: + if c.execute('SELECT COUNT(*) n FROM earthquakes').fetchone()['n']>0: return + rng=random.Random(7); now_ms=int(time.time()*1000) + places=['Bootstrap demo, California','Bootstrap demo, Alaska','Bootstrap demo, Japan','Bootstrap demo, Chile','Bootstrap demo, Indonesia'] + rows=[] + for i in range(180): + lat=rng.uniform(-55,65); lon=rng.uniform(-170,170); mag=round(rng.uniform(1,6.4),1) + rows.append({'id':f'bootstrap-{i}','time_ms':now_ms-i*3600_000,'updated_ms':now_ms,'place':rng.choice(places),'magnitude':mag,'longitude':round(lon,3),'latitude':round(lat,3),'depth':round(rng.uniform(2,140),1),'tsunami':0,'significance':int(mag*100),'event_type':'earthquake','region':rough_region(lat,lon,rng.choice(places))}) + upsert_earthquakes(db_path, rows, 'bootstrap_demo', True) + +def query_earthquakes(db_path: str, filters: dict[str,Any], limit: int, sort='time_desc') -> list[dict[str,Any]]: + where=[]; args=[] + for key,col,op in [('mag_min','magnitude','>='),('mag_max','magnitude','<='),('lat_min','latitude','>='),('lat_max','latitude','<='),('lon_min','longitude','>='),('lon_max','longitude','<=')]: + if filters.get(key) is not None: where.append(f'{col} {op} ?'); args.append(filters[key]) + if filters.get('start_time') is not None: where.append('time_ms >= ?'); args.append(filters['start_time']) + if filters.get('end_time') is not None: where.append('time_ms <= ?'); args.append(filters['end_time']) + sql='SELECT * FROM earthquakes '+(('WHERE '+ ' AND '.join(where)) if where else '') + order={'time_asc':'time_ms ASC','magnitude_desc':'magnitude DESC','magnitude_asc':'magnitude ASC'}.get(sort,'time_ms DESC') + sql += f' ORDER BY {order} LIMIT ?'; args.append(limit) + with connect(db_path) as c: return [dict(r) for r in c.execute(sql,args).fetchall()] + +def count_records(db_path: str) -> int: + with connect(db_path) as c: return c.execute('SELECT COUNT(*) n FROM earthquakes').fetchone()['n'] + +def all_records(db_path: str, limit=10000) -> list[dict[str,Any]]: return query_earthquakes(db_path, {}, limit, 'time_asc') diff --git a/app/services/model.py b/app/services/model.py new file mode 100644 index 0000000..534aba0 --- /dev/null +++ b/app/services/model.py @@ -0,0 +1,72 @@ +from __future__ import annotations +import json, math +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from app.utils import DISCLAIMER, haversine_km +FEATURE_NAMES=['event_count_24h','event_count_7d','max_mag_7d','avg_mag_7d','count_m25_7d','count_m45_7d','mean_depth_7d','time_since_last_hours','recent_acceleration','aftershock_large_nearby','lat','lon','hour','day_of_week'] + +def nearby(rows, lat, lon, radius): + out=[] + for r in rows: + if r.get('latitude') is None or r.get('longitude') is None: continue + d=haversine_km(lat,lon,float(r['latitude']),float(r['longitude'])) + if d<=radius: + x=dict(r); x['distance_km']=round(d,1); out.append(x) + return sorted(out,key=lambda r:r.get('time_ms') or 0, reverse=True) +def feature_vector(rows, lat, lon, radius, hours, min_mag): + now=max([r.get('time_ms') or 0 for r in rows], default=int(datetime.now(timezone.utc).timestamp()*1000)); near=nearby(rows,lat,lon,radius) + def win(h): return [r for r in near if now-(r.get('time_ms') or 0)<=h*3600_000] + w24=win(24); w7=win(24*7); mags=[r.get('magnitude') or 0 for r in w7]; depths=[r.get('depth') or 0 for r in w7] + last=max([r.get('time_ms') or 0 for r in near], default=0) + f={'event_count_24h':len(w24),'event_count_7d':len(w7),'max_mag_7d':max(mags) if mags else 0,'avg_mag_7d':(sum(mags)/len(mags)) if mags else 0,'count_m25_7d':sum(1 for m in mags if m>=2.5),'count_m45_7d':sum(1 for m in mags if m>=4.5),'mean_depth_7d':(sum(depths)/len(depths)) if depths else 0,'time_since_last_hours':round((now-last)/3600_000,2) if last else 999,'recent_acceleration':len(w24)-((len(w7)-len(w24))/6 if len(w7)>len(w24) else 0),'aftershock_large_nearby':1 if any((r.get('magnitude') or 0)>=5 for r in w24) else 0,'lat':lat,'lon':lon,'hour':datetime.fromtimestamp(now/1000,tz=timezone.utc).hour,'day_of_week':datetime.fromtimestamp(now/1000,tz=timezone.utc).weekday()} + return f, near[:10] +def heuristic(features): + score=0 + score+=min(features['event_count_24h']*4,25)+min(features['event_count_7d']*1.4,25)+min(features['max_mag_7d']*7,28)+features['count_m45_7d']*8 + if features['aftershock_large_nearby']: score+=15 + if features['time_since_last_hours']<6: score+=10 + score=max(0,min(100,score)); return score +def bucket(score): return 'high' if score>=75 else 'elevated' if score>=50 else 'moderate' if score>=25 else 'low' +def load_artifact(path): + p=Path(path) + if p.exists(): + try: + import joblib + return joblib.load(p) + except Exception: return None + return None +def predict(rows, model_path, lat, lon, radius, hours, min_mag): + feats, near=feature_vector(rows,lat,lon,radius,hours,min_mag); score=heuristic(feats); confidence='low'; source='heuristic_baseline' + art=load_artifact(model_path) + if art and art.get('model'): + try: + import pandas as pd + proba=float(art['model'].predict_proba(pd.DataFrame([feats])[FEATURE_NAMES])[0][1]); score=round((score*0.45+proba*100*0.55),2); confidence='moderate'; source='supervised_model_plus_heuristic' + except Exception: pass + top=sorted(feats.items(), key=lambda kv: abs(float(kv[1])) if isinstance(kv[1],(int,float)) else 0, reverse=True)[:6] + return {'risk_score':round(score,2),'estimated_probability':round(score/100,3),'probability_bucket':bucket(score),'expected_activity_count':round(feats['event_count_24h']*(hours/24)+0.2*feats['event_count_7d'],2),'model_confidence':confidence,'model_source':source,'top_contributing_features':[{'feature':k,'value':v} for k,v in top],'nearby_recent_events':near[:8],'explanation':'Risk score blends recent nearby activity, magnitude/depth patterns, aftershock heuristics, and an optional trained classifier when enough cached data exists.','warning':DISCLAIMER} +def train(rows, model_path): + if len(rows)<80: return {'trained':False,'reason':'not_enough_rows','training_rows':len(rows)} + try: + import joblib, numpy as np, pandas as pd + from sklearn.ensemble import HistGradientBoostingClassifier + from sklearn.metrics import accuracy_score, confusion_matrix, f1_score, precision_score, recall_score, roc_auc_score + except Exception as exc: + return {'trained':False,'reason':'ml_dependencies_unavailable','error':str(exc),'training_rows':len(rows)} + df=pd.DataFrame(rows).sort_values('time_ms'); samples=[]; target=[] + for _,r in df.iterrows(): + lat=float(r.latitude); lon=float(r.longitude); t=int(r.time_ms); hist=[x for x in rows if (x.get('time_ms') or 0) <= t] + feats,_=feature_vector(hist,lat,lon,250,72,2.5); future=[x for x in rows if t < (x.get('time_ms') or 0) <= t+72*3600_000 and haversine_km(lat,lon,x.get('latitude'),x.get('longitude'))<=250 and (x.get('magnitude') or 0)>=2.5] + samples.append(feats); target.append(1 if future else 0) + if len(set(target))<2: return {'trained':False,'reason':'single_class_target','training_rows':len(target)} + X=pd.DataFrame(samples)[FEATURE_NAMES]; y=np.array(target); split=int(len(X)*0.75); Xtr,Xte=X.iloc[:split],X.iloc[split:]; ytr,yte=y[:split],y[split:] + model=HistGradientBoostingClassifier(max_iter=80, learning_rate=0.08).fit(Xtr,ytr); pred=model.predict(Xte); proba=model.predict_proba(Xte)[:,1] + metrics={'accuracy':float(accuracy_score(yte,pred)),'precision':float(precision_score(yte,pred,zero_division=0)),'recall':float(recall_score(yte,pred,zero_division=0)),'f1':float(f1_score(yte,pred,zero_division=0)),'confusion_matrix':confusion_matrix(yte,pred).tolist()} + if len(set(yte))>1: metrics['roc_auc']=float(roc_auc_score(yte,proba)) + meta={'model_version':'2026.06','trained_at':datetime.now(timezone.utc).isoformat(),'training_rows':len(X),'metrics':metrics,'feature_names':FEATURE_NAMES,'artifact_path':model_path} + Path(model_path).parent.mkdir(parents=True,exist_ok=True); joblib.dump({'model':model,'metadata':meta}, model_path); Path(model_path+'.json').write_text(json.dumps(meta,indent=2)) + return {'trained':True, **meta} +def status(model_path): + art=load_artifact(model_path); meta=(art or {}).get('metadata',{}) + return {'model_exists':bool(art),'model_version':meta.get('model_version'),'trained_at':meta.get('trained_at'),'training_rows':meta.get('training_rows',0),'metrics':meta.get('metrics',{}),'feature_names':meta.get('feature_names',FEATURE_NAMES),'artifact_path':model_path} diff --git a/app/services/usgs_client.py b/app/services/usgs_client.py new file mode 100644 index 0000000..a448c6e --- /dev/null +++ b/app/services/usgs_client.py @@ -0,0 +1,41 @@ +from __future__ import annotations +import logging +from typing import Any +from app.utils import rough_region, safe_float +log=logging.getLogger(__name__) + +def parse_geojson_features(payload: dict[str, Any]) -> list[dict[str, Any]]: + if not isinstance(payload, dict) or not isinstance(payload.get('features'), list): + raise ValueError('Invalid USGS GeoJSON payload') + rows=[] + for f in payload.get('features', []): + if not isinstance(f, dict): continue + props=f.get('properties') or {}; geom=f.get('geometry') or {}; coords=geom.get('coordinates') or [] + if len(coords) < 2 or f.get('id') is None: continue + lon=safe_float(coords[0]); lat=safe_float(coords[1]); depth=safe_float(coords[2] if len(coords)>2 else 0, 0.0); mag=safe_float(props.get('mag')) + if lon is None or lat is None or mag is None or not (-180<=lon<=180) or not (-90<=lat<=90): continue + place=str(props.get('place') or 'Unknown') + row={ + 'id': str(f.get('id')), 'time_ms': int(props.get('time') or 0), 'updated_ms': int(props.get('updated') or 0), + 'place': place, 'magnitude': float(mag), 'longitude': float(lon), 'latitude': float(lat), 'depth': float(depth or 0), + 'alert': props.get('alert'), 'tsunami': int(props.get('tsunami') or 0), 'significance': props.get('sig'), 'felt': props.get('felt'), + 'cdi': props.get('cdi'), 'mmi': props.get('mmi'), 'status': props.get('status'), 'event_type': props.get('type'), + 'network': props.get('net'), 'code': props.get('code'), 'url': props.get('url'), 'detail_url': props.get('detail'), + 'mag_type': props.get('magType'), 'gap': props.get('gap'), 'dmin': props.get('dmin'), 'rms': props.get('rms'), 'nst': props.get('nst'), + } + row['region']=rough_region(row['latitude'], row['longitude'], place) + rows.append(row) + return rows + +def fetch_feed(url: str, timeout: int=15) -> list[dict[str, Any]]: + try: + import requests + resp=requests.get(url, timeout=timeout, headers={'User-Agent':'earthquake-risk-dashboard/1.0'}) + resp.raise_for_status() + return parse_geojson_features(resp.json()) + except ImportError: + import json + from urllib.request import Request, urlopen + req=Request(url, headers={'User-Agent':'earthquake-risk-dashboard/1.0'}) + with urlopen(req, timeout=timeout) as resp: + return parse_geojson_features(json.loads(resp.read().decode('utf-8'))) diff --git a/app/utils.py b/app/utils.py new file mode 100644 index 0000000..4dae24d --- /dev/null +++ b/app/utils.py @@ -0,0 +1,49 @@ +from __future__ import annotations +from datetime import datetime, timezone +from math import asin, cos, radians, sin, sqrt +from typing import Any +from flask import request + +DISCLAIMER = ('Experimental probabilistic seismic activity forecast only. This app does not predict exact earthquake times, locations, or magnitudes; use official emergency sources for safety decisions.') + +def utc_now_ms() -> int: return int(datetime.now(timezone.utc).timestamp()*1000) +def iso_from_ms(ms: int | None) -> str | None: + if not ms: return None + return datetime.fromtimestamp(ms/1000, tz=timezone.utc).isoformat() +def parse_time_ms(value: str | None) -> int | None: + if not value: return None + try: + if value.isdigit(): + v=int(value); return v if v>10_000_000_000 else v*1000 + return int(datetime.fromisoformat(value.replace('Z','+00:00')).timestamp()*1000) + except Exception: return None + +def safe_float(v: Any, default: float | None=None) -> float | None: + try: + if v is None or v == '': return default + f=float(v) + return f if f == f and abs(f) != float('inf') else default + except Exception: return default + +def safe_int(v: Any, default: int=0, lo: int | None=None, hi: int | None=None) -> int: + try: n=int(v) + except Exception: n=default + if lo is not None: n=max(lo,n) + if hi is not None: n=min(hi,n) + return n + +def bool_arg(name: str, default: bool=False) -> bool: + raw=request.args.get(name) + if raw is None: return default + return raw.lower() in {'1','true','yes','on'} + +def haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float: + r=6371.0; p1,p2=radians(lat1),radians(lat2); dp=radians(lat2-lat1); dl=radians(lon2-lon1) + a=sin(dp/2)**2+cos(p1)*cos(p2)*sin(dl/2)**2 + return 2*r*asin(sqrt(a)) + +def rough_region(lat: float | None, lon: float | None, place: str='') -> str: + if place and ',' in place: return place.split(',')[-1].strip()[:80] + if lat is None or lon is None: return 'Unknown' + ns='Northern' if lat>=0 else 'Southern'; ew='Eastern' if lon>=0 else 'Western' + return f'{ns} {ew} Hemisphere' diff --git a/main.py b/main.py index 65c70ae..693b134 100644 --- a/main.py +++ b/main.py @@ -1,294 +1,5 @@ -import json import os -import random -import sqlite3 -import time -from pathlib import Path -from typing import List, Dict, Any -from urllib.error import URLError -from urllib.request import urlopen - -from flask import Flask, jsonify, render_template, request - -app = Flask(__name__) - -BASE_DIR = Path(__file__).resolve().parent -DATA_DIR = BASE_DIR / "data" -DB_PATH = Path(os.getenv("SQLITE_DB_PATH", DATA_DIR / "earthquakes.db")) -USGS_FEED_URL = os.getenv( - "USGS_FEED_URL", - "https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.geojson", -) -CACHE_TTL_SECONDS = int(os.getenv("CACHE_TTL_SECONDS", "300")) - - -def init_db() -> None: - DB_PATH.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(DB_PATH) as conn: - conn.execute( - """ - CREATE TABLE IF NOT EXISTS earthquakes ( - id TEXT PRIMARY KEY, - time_ms INTEGER, - place TEXT, - magnitude REAL, - latitude REAL, - longitude REAL, - depth REAL, - updated_at INTEGER - ) - """ - ) - conn.execute( - """ - CREATE TABLE IF NOT EXISTS metadata ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ) - """ - ) - conn.commit() - - -def _set_metadata(conn: sqlite3.Connection, key: str, value: str) -> None: - conn.execute( - "INSERT INTO metadata(key, value) VALUES(?, ?) " - "ON CONFLICT(key) DO UPDATE SET value=excluded.value", - (key, value), - ) - - -def _get_metadata(conn: sqlite3.Connection, key: str, default: str = "") -> str: - row = conn.execute("SELECT value FROM metadata WHERE key=?", (key,)).fetchone() - return row[0] if row else default - - -def parse_geojson_features(payload: Dict[str, Any]) -> List[Dict[str, Any]]: - features = payload.get("features", []) - parsed = [] - for f in features: - props = f.get("properties", {}) - coords = f.get("geometry", {}).get("coordinates", [None, None, None]) - quake_id = f.get("id") - if quake_id is None or coords[0] is None or coords[1] is None: - continue - magnitude = props.get("mag") - if magnitude is None: - continue - parsed.append( - { - "id": str(quake_id), - "time_ms": int(props.get("time") or 0), - "place": str(props.get("place") or "Unknown"), - "magnitude": float(magnitude), - "longitude": float(coords[0]), - "latitude": float(coords[1]), - "depth": float(coords[2] or 0.0), - } - ) - return parsed - - -def fetch_live_usgs() -> List[Dict[str, Any]]: - with urlopen(USGS_FEED_URL, timeout=15) as resp: - payload = json.loads(resp.read().decode("utf-8")) - return parse_geojson_features(payload) - - -def replace_cache(rows: List[Dict[str, Any]]) -> None: - now = int(time.time()) - with sqlite3.connect(DB_PATH) as conn: - conn.execute("DELETE FROM earthquakes") - conn.executemany( - """ - INSERT INTO earthquakes(id, time_ms, place, magnitude, latitude, longitude, depth, updated_at) - VALUES(?,?,?,?,?,?,?,?) - """, - [ - ( - r["id"], - r["time_ms"], - r["place"], - r["magnitude"], - r["latitude"], - r["longitude"], - r["depth"], - now, - ) - for r in rows - ], - ) - _set_metadata(conn, "last_refresh_epoch", str(now)) - _set_metadata(conn, "last_refresh_source", "live") - conn.commit() - - -def refresh_cache_if_needed(force: bool = False) -> str: - with sqlite3.connect(DB_PATH) as conn: - last_refresh = int(_get_metadata(conn, "last_refresh_epoch", "0") or "0") - - is_stale = int(time.time()) - last_refresh > CACHE_TTL_SECONDS - if not force and not is_stale: - return "cache" - - try: - rows = fetch_live_usgs() - if rows: - replace_cache(rows) - return "live" - except (URLError, TimeoutError, ValueError, OSError): - pass - - with sqlite3.connect(DB_PATH) as conn: - count = conn.execute("SELECT COUNT(*) FROM earthquakes").fetchone()[0] - if count == 0: - # Ensure UI still renders if first live fetch fails in restricted environments. - conn.commit() - seed_bootstrap_data_if_empty() - return "bootstrap_cache" - _set_metadata(conn, "last_refresh_source", "fallback_cache") - conn.commit() - return "fallback_cache" - - -def query_quakes( - mag_min: float, - mag_max: float, - lat_min: float, - lat_max: float, - lon_min: float, - lon_max: float, -): - with sqlite3.connect(DB_PATH) as conn: - rows = conn.execute( - """ - SELECT id, time_ms, place, magnitude, latitude, longitude, depth - FROM earthquakes - WHERE magnitude BETWEEN ? AND ? - AND latitude BETWEEN ? AND ? - AND longitude BETWEEN ? AND ? - ORDER BY time_ms DESC - """, - (mag_min, mag_max, lat_min, lat_max, lon_min, lon_max), - ).fetchall() - source = _get_metadata(conn, "last_refresh_source", "cache") - refreshed = int(_get_metadata(conn, "last_refresh_epoch", "0") or "0") - - payload = [ - { - "id": r[0], - "time_ms": r[1], - "place": r[2], - "magnitude": r[3], - "latitude": r[4], - "longitude": r[5], - "depth": r[6], - } - for r in rows - ] - return payload, source, refreshed - -def seed_bootstrap_data_if_empty() -> None: - with sqlite3.connect(DB_PATH) as conn: - existing = conn.execute("SELECT COUNT(*) FROM earthquakes").fetchone()[0] - if existing > 0: - return - - rng = random.Random(7) - now = int(time.time()) - rows = [] - for i in range(220): - rows.append( - ( - f"bootstrap-{i}", - (now - i * 1800) * 1000, - "Bootstrap cached sample", - round(rng.uniform(1.0, 6.8), 1), - round(rng.uniform(-70, 70), 3), - round(rng.uniform(-170, 170), 3), - round(rng.uniform(1, 120), 1), - now, - ) - ) - - conn.executemany( - """ - INSERT INTO earthquakes(id, time_ms, place, magnitude, latitude, longitude, depth, updated_at) - VALUES(?,?,?,?,?,?,?,?) - """, - rows, - ) - _set_metadata(conn, "last_refresh_epoch", str(now)) - _set_metadata(conn, "last_refresh_source", "bootstrap_cache") - conn.commit() - - -def parse_float(name: str, default: float) -> float: - raw = request.values.get(name, default) - try: - return float(raw) - except (TypeError, ValueError): - return default - - -@app.route("/") -@app.route("/dashboard") -def dashboard(): - return render_template("dashboard.html") - - -@app.route("/globe") -def globe(): - return render_template("globe.html") - - -@app.route("/api/earthquakes") -def api_earthquakes(): - mag_min = parse_float("mag_min", 0.0) - mag_max = parse_float("mag_max", 10.0) - lat1 = parse_float("lat1", -90.0) - lat2 = parse_float("lat2", 90.0) - lon1 = parse_float("lon1", -180.0) - lon2 = parse_float("lon2", 180.0) - force_refresh = request.args.get("refresh", "0") == "1" - - mag_min, mag_max = sorted((mag_min, mag_max)) - lat_min, lat_max = sorted((lat1, lat2)) - lon_min, lon_max = sorted((lon1, lon2)) - - refresh_result = refresh_cache_if_needed(force=force_refresh) - rows, source, refreshed = query_quakes(mag_min, mag_max, lat_min, lat_max, lon_min, lon_max) - - return jsonify( - { - "meta": { - "source": source, - "refresh_result": refresh_result, - "last_refresh_epoch": refreshed, - "count": len(rows), - "filters": { - "mag_min": mag_min, - "mag_max": mag_max, - "lat_min": lat_min, - "lat_max": lat_max, - "lon_min": lon_min, - "lon_max": lon_max, - }, - }, - "data": rows, - } - ) - - -@app.route("/health") -def health(): - return jsonify({"status": "ok"}) - - -init_db() -refresh_cache_if_needed(force=False) -seed_bootstrap_data_if_empty() - -if __name__ == "__main__": - port = int(os.getenv("PORT", 5000)) - app.run(debug=False, host="0.0.0.0", port=port) +from app import create_app +app = create_app() +if __name__ == '__main__': + app.run(host='0.0.0.0', port=int(os.getenv('PORT', '5000')), debug=False) diff --git a/render.yaml b/render.yaml index 5a54538..aa8ae55 100644 --- a/render.yaml +++ b/render.yaml @@ -5,11 +5,18 @@ services: plan: free buildCommand: pip install -r requirements.txt startCommand: gunicorn main:app --bind 0.0.0.0:$PORT + healthCheckPath: /health autoDeploy: true envVars: - key: PYTHON_VERSION - value: 3.11.9 + value: 3.11.15 - key: CACHE_TTL_SECONDS value: 300 - - key: USGS_FEED_URL - value: https://earthquake.usgs.gov/earthquakes/feed/v1.0/summary/all_day.geojson + - key: DEFAULT_FEED_WINDOW + value: day + - key: ENABLE_BOOTSTRAP_DATA + value: true + - key: ENABLE_MODEL_TRAINING_ON_STARTUP + value: false + - key: MAX_API_LIMIT + value: 2000 diff --git a/requirements.txt b/requirements.txt index f4696b2..7525fd9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,8 @@ -Flask>=3.0.0 -gunicorn>=21.2.0 +Flask>=3.0,<4.0 +gunicorn>=21.2,<23 +requests>=2.32,<3 +pandas>=2.2,<3 +numpy>=1.26,<3 +scikit-learn>=1.4,<2 +joblib>=1.3,<2 +pytest>=8.0,<9 diff --git a/static/css/app.css b/static/css/app.css new file mode 100644 index 0000000..346aeb2 --- /dev/null +++ b/static/css/app.css @@ -0,0 +1 @@ +:root{--bg:#f6f8fc;--panel:#fff;--text:#122033;--muted:#5c6b82;--accent:#3867ff;--danger:#ef4444;--warn:#f59e0b;--ok:#10b981;--border:#dce4f0}html[data-theme=dark]{--bg:#09111f;--panel:#111c2e;--text:#e8eef8;--muted:#98a9c2;--accent:#7aa2ff;--border:#22324a}*{box-sizing:border-box}body{margin:0;font-family:Inter,system-ui,Segoe UI,Arial;background:radial-gradient(circle at top left,rgba(56,103,255,.18),transparent 38%),var(--bg);color:var(--text)}.nav{position:sticky;top:0;z-index:20;display:flex;gap:1rem;align-items:center;flex-wrap:wrap;padding:.9rem 1.2rem;background:rgba(10,16,29,.82);backdrop-filter:blur(14px);border-bottom:1px solid var(--border)}.nav a{color:#e8eef8;text-decoration:none;font-weight:700}.brand{font-size:1.25rem;margin-right:auto}.ghost,.btn,button{border:0;border-radius:10px;padding:.65rem .9rem;font-weight:800;cursor:pointer}.ghost{background:transparent;color:#e8eef8;border:1px solid #53627a}.btn,button{background:var(--accent);color:white}main{max-width:1320px;margin:0 auto;padding:1.2rem}.hero,.card{background:linear-gradient(180deg,rgba(255,255,255,.06),transparent),var(--panel);border:1px solid var(--border);border-radius:20px;padding:1.1rem;box-shadow:0 18px 45px rgba(0,0,0,.18);margin-bottom:1rem}.hero{display:flex;justify-content:space-between;gap:1rem;flex-wrap:wrap}.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(260px,1fr));gap:1rem}.kpi{border-left:4px solid var(--accent)}.kpi b{font-size:2rem}.muted{color:var(--muted)}label{display:block;font-weight:800;margin-bottom:.25rem}input,select{width:100%;padding:.65rem;border-radius:10px;border:1px solid var(--border);background:var(--bg);color:var(--text)}.controls{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:.8rem;align-items:end}.table{width:100%;border-collapse:collapse}.table td,.table th{padding:.55rem;border-bottom:1px solid var(--border);text-align:left}.pill{display:inline-block;border-radius:999px;padding:.25rem .55rem;background:rgba(122,162,255,.18)}.warning{border-left:4px solid var(--warn)}#map{height:72vh;border-radius:18px}.score{font-size:3rem;font-weight:900}.low{color:var(--ok)}.moderate{color:var(--warn)}.elevated,.high{color:var(--danger)}@media(max-width:700px){.nav{position:static}.hero{display:block}.score{font-size:2.2rem}} diff --git a/static/js/dashboard.js b/static/js/dashboard.js new file mode 100644 index 0000000..ac0ebd4 --- /dev/null +++ b/static/js/dashboard.js @@ -0,0 +1 @@ +let charts=[], lastRows=[];const $=id=>document.getElementById(id);function chart(id,type,labels,data,label){const c=new Chart($(id),{type,data:{labels,datasets:[{label,data,backgroundColor:['#10b981','#f59e0b','#fb923c','#ef4444','#8b5cf6'],borderColor:'#7aa2ff'}]},options:{responsive:true}});charts.push(c)}function csv(rows){const cols=['id','time_ms','place','magnitude','latitude','longitude','depth','alert','tsunami','url'];return [cols.join(','),...rows.map(r=>cols.map(c=>`"${String(r[c]??'').replaceAll('"','""')}"`).join(','))].join('\n')}async function load(force=false){charts.forEach(c=>c.destroy());charts=[];const p=new URLSearchParams({feed:$('feed').value,mag_min:$('magMin').value,limit:$('limit').value,refresh:force?'1':'0'});const [eq,sum,fc]=await Promise.all([fetch('/api/earthquakes?'+p).then(r=>r.json()),fetch('/api/summary?'+p).then(r=>r.json()),fetch('/api/forecast?lat=37.7749&lon=-122.4194&radius_km=300').then(r=>r.json())]);lastRows=eq.data||[];const s=sum.summary;$('total').textContent=s.total_events;$('maxmag').textContent=s.max_magnitude??'-';$('avgmag').textContent=s.average_magnitude??'-';$('tsunami').textContent=s.tsunami_count;$('source').textContent=`Source ${eq.source}; count ${eq.count}; ${eq.warnings?.join(' ')||''}`;chart('trend','line',s.hourly_trend.map(x=>x.period).slice(-24),s.hourly_trend.map(x=>x.count).slice(-24),'Events');chart('magChart','doughnut',Object.keys(s.magnitude_distribution),Object.values(s.magnitude_distribution),'Magnitude');chart('depthChart','bar',Object.keys(s.depth_distribution),Object.values(s.depth_distribution),'Depth');$('regions').innerHTML=s.top_regions.map(x=>`
  • ${x[0]} ${x[1]}
    • `).join('');$('events').innerHTML=lastRows.filter(r=>(r.magnitude||0)>=4.5||r.tsunami).slice(0,10).map(r=>`M${r.magnitude}${r.place}${new Date(r.time_ms).toLocaleString()}`).join('')||'No significant events in current filter.';$('forecastCard').innerHTML=`
    ${fc.risk_score}
    ${fc.probability_bucket}

    ${fc.explanation}

    `} $('apply').onclick=()=>load(false);$('refresh').onclick=()=>load(true);$('export').onclick=()=>{const a=document.createElement('a');a.href=URL.createObjectURL(new Blob([csv(lastRows)],{type:'text/csv'}));a.download='earthquakes.csv';a.click()};load();setInterval(()=>load(false),120000); diff --git a/static/js/forecast.js b/static/js/forecast.js new file mode 100644 index 0000000..944582b --- /dev/null +++ b/static/js/forecast.js @@ -0,0 +1 @@ +function preset(a,b){lat.value=a;lon.value=b;run()}async function run(){const p=new URLSearchParams({lat:lat.value,lon:lon.value,radius_km:radius.value,hours:hours.value,min_magnitude:minmag.value});const r=await fetch('/api/forecast?'+p).then(x=>x.json());score.textContent=r.risk_score;score.className='score '+r.probability_bucket;bucket.textContent=`${r.probability_bucket} · confidence ${r.model_confidence}`;explain.textContent=r.explanation;warn.textContent=r.warning;features.innerHTML=r.top_contributing_features.map(f=>`
  • ${f.feature}: ${f.value}
    • `).join('');nearby.innerHTML=r.nearby_recent_events.map(q=>`M${q.magnitude}${q.place}${q.distance_km} km`).join('')||'No nearby cached events.'}document.getElementById('run').onclick=run;run(); diff --git a/static/js/globe.js b/static/js/globe.js index 6d6e7cb..1982502 100644 --- a/static/js/globe.js +++ b/static/js/globe.js @@ -1,4 +1,5 @@ (() => { + if (typeof Globe === 'undefined') { const b=document.getElementById('statusBanner'); if(b) b.textContent='3D globe library failed to load. Please check the CDN connection or use the 2D map.'; return; } const globeContainer = document.getElementById('globeViz'); const statusBanner = document.getElementById('statusBanner'); const stats = document.getElementById('stats'); diff --git a/static/js/map.js b/static/js/map.js new file mode 100644 index 0000000..6361b14 --- /dev/null +++ b/static/js/map.js @@ -0,0 +1 @@ +const map=L.map('map').setView([20,0],2);L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png',{attribution:'© OpenStreetMap'}).addTo(map);let layer=L.layerGroup().addTo(map);function color(m){return m>=6?'#ef4444':m>=4?'#f97316':m>=2?'#f59e0b':'#10b981'}async function load(useBounds=false){layer.clearLayers();const p=new URLSearchParams({feed:feed.value,mag_min:mag.value,limit:1500});if(useBounds){const b=map.getBounds();p.set('lat1',b.getSouth());p.set('lat2',b.getNorth());p.set('lon1',b.getWest());p.set('lon2',b.getEast())}const payload=await fetch('/api/earthquakes?'+p).then(r=>r.json());(payload.data||[]).forEach(q=>{L.circleMarker([q.latitude,q.longitude],{radius:4+(q.magnitude||0)*2,color:color(q.magnitude||0),fillOpacity:.65}).bindPopup(`${q.place}
    M ${q.magnitude} · ${q.depth} km
    ${new Date(q.time_ms).toLocaleString()}
    ${q.alert?`Alert: ${q.alert}
    `:''}USGS detail`).addTo(layer)});status.textContent=`${payload.count} events · ${payload.source}`}document.getElementById('load').onclick=()=>load(false);document.getElementById('bounds').onclick=()=>load(true);load(); diff --git a/templates/about.html b/templates/about.html new file mode 100644 index 0000000..8273ca9 --- /dev/null +++ b/templates/about.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block title %}About{% endblock %}{% block content %}

    About QuakeRisk

    QuakeRisk is a Flask, SQLite, Chart.js, Leaflet, and globe.gl application for monitoring USGS GeoJSON earthquake feeds.

    Scientific limitation

    Earthquake prediction is not scientifically reliable. This project only provides probabilistic earthquake forecasting, regional risk scoring, magnitude likelihood estimation, and aftershock/seismic activity trend analysis for education and situational awareness.

    Deployment

    Render can run the app with gunicorn main:app --bind 0.0.0.0:$PORT. SQLite works by default, but free instances may lose local cache data on restart unless a persistent disk or external database is configured.

    Routes

    {% endblock %} diff --git a/templates/base.html b/templates/base.html new file mode 100644 index 0000000..da3c866 --- /dev/null +++ b/templates/base.html @@ -0,0 +1 @@ +{% block title %}Earthquake Monitor{% endblock %}{% block head %}{% endblock %}
    {% block content %}{% endblock %}
    {% block scripts %}{% endblock %} diff --git a/templates/dashboard.html b/templates/dashboard.html index c7ee59d..53e7c35 100644 --- a/templates/dashboard.html +++ b/templates/dashboard.html @@ -1,148 +1 @@ - - - - - - Live Earthquake Dashboard - - - - -
    -
    -
    -

    Live Earthquake Dashboard

    -

    Source: USGS all_day feed (with cache fallback for reliability on Render)

    -
    - Open 3D Globe View -
    - -
    -
    -
    -
    -
    -
    - -
    -
    -
    -
    -
    -
    -
    - -
    -

    Total Points

    0

    -

    Average Magnitude

    0

    -

    Max Magnitude

    0

    -
    - -
    -
    -
    -
    -
    -
    -
    - - - - +{% extends 'base.html' %}{% block title %}Earthquake Analytics Dashboard{% endblock %}{% block head %}{% endblock %}{% block content %}

    Earthquake Monitoring Dashboard

    Live USGS feeds with resilient SQLite cache, analytics, and honest probabilistic risk scoring.

    Open Map Experimental Forecast
    Total events
    0
    Max magnitude
    -
    Average magnitude
    -
    Tsunami flags
    0

    Hourly trend

    Magnitude distribution

    Depth distribution

    Experimental model forecast

    Loading...

    Not deterministic earthquake prediction.

    Top regions

      Significant events

      {% endblock %}{% block scripts %}{% endblock %} diff --git a/templates/forecast.html b/templates/forecast.html new file mode 100644 index 0000000..4e82169 --- /dev/null +++ b/templates/forecast.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block title %}Probabilistic Forecast{% endblock %}{% block head %}{% endblock %}{% block content %}

      Experimental Probabilistic Forecast

      Regional risk scoring and seismic activity trend analysis. This is not deterministic earthquake prediction.

      Risk score

      --

      Contributing features

        Nearby recent events

        {% endblock %}{% block scripts %}{% endblock %} diff --git a/templates/globe.html b/templates/globe.html index 21155ce..29d8d4d 100644 --- a/templates/globe.html +++ b/templates/globe.html @@ -14,7 +14,7 @@

        Live Earthquake 3D Globe

        Powered by /api/earthquakes with live refresh and cache fallback.

        - Dashboard + Dashboard2D MapForecast
        diff --git a/templates/map.html b/templates/map.html new file mode 100644 index 0000000..0230e14 --- /dev/null +++ b/templates/map.html @@ -0,0 +1 @@ +{% extends 'base.html' %}{% block title %}Earthquake Map{% endblock %}{% block head %}{% endblock %}{% block content %}

        2D Earthquake Map

        Markers are sized and colored by magnitude. Use the viewport button to query map bounds.

        {% endblock %}{% block scripts %}{% endblock %} diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..b6efa15 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,5 @@ +import sys +from pathlib import Path +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) diff --git a/tests/test_app.py b/tests/test_app.py new file mode 100644 index 0000000..ed04f3e --- /dev/null +++ b/tests/test_app.py @@ -0,0 +1,55 @@ +import os +import pytest + +SAMPLE={"type":"FeatureCollection","features":[{"type":"Feature","id":"abc","properties":{"mag":2.7,"place":"10 km S of Test, California","time":1700000000000,"updated":1700000001000,"sig":110,"tsunami":0,"net":"us","code":"abc","type":"earthquake"},"geometry":{"type":"Point","coordinates":[-122.1,37.2,8.5]}}]} + +@pytest.fixture() +def client(tmp_path, monkeypatch): + monkeypatch.setenv('SQLITE_DB_PATH', str(tmp_path/'test.db')) + monkeypatch.setenv('ENABLE_BOOTSTRAP_DATA','false') + from app import create_app + app=create_app(); app.config['TESTING']=True + from app.services.cache import upsert_earthquakes + from app.services.usgs_client import parse_geojson_features + upsert_earthquakes(app.config['SQLITE_DB_PATH'], parse_geojson_features(SAMPLE), 'test') + return app.test_client() + +def test_geojson_parsing(): + from app.services.usgs_client import parse_geojson_features + rows=parse_geojson_features(SAMPLE) + assert rows[0]['id']=='abc' + assert rows[0]['magnitude']==2.7 + assert rows[0]['latitude']==37.2 + assert rows[0]['region']=='California' + +def test_invalid_geojson(): + from app.services.usgs_client import parse_geojson_features + with pytest.raises(ValueError): parse_geojson_features({'bad': []}) + +def test_cache_upsert(tmp_path): + from app.services.cache import init_db, upsert_earthquakes, count_records + db=str(tmp_path/'x.db'); init_db(db) + row={'id':'1','time_ms':1,'updated_ms':1,'place':'A','magnitude':1.0,'longitude':1,'latitude':1,'depth':2} + upsert_earthquakes(db,[row],'test'); row['magnitude']=3.0; upsert_earthquakes(db,[row],'test') + assert count_records(db)==1 + +def test_health(client): assert client.get('/health').json['status']=='ok' +def test_earthquakes_api(client): + r=client.get('/api/earthquakes?mag_min=1&limit=5') + assert r.status_code==200 and r.json['count']>=1 and isinstance(r.json['data'], list) +def test_query_param_parsing(client): + r=client.get('/api/earthquakes?mag_min=bad&lat1=99&limit=999999') + assert r.status_code==200 + assert r.json['filters']['lat_max'] == 90 + +def test_summary(client): + r=client.get('/api/summary') + assert r.status_code==200 and 'total_events' in r.json['summary'] + +def test_forecast_fallback(client): + r=client.get('/api/forecast?lat=37.2&lon=-122.1&radius_km=50') + assert r.status_code==200 and 'risk_score' in r.json and 'does not predict exact' in r.json['warning'] + +def test_model_status(client): + r=client.get('/api/model/status') + assert r.status_code==200 and 'model_exists' in r.json