B2: tolerant JSON extraction from Claude responses + graceful fallback
_extract_json_payload handles a json/JSON/bare fence, raw JSON and JSON embedded in prose; any unparseable response or contract violation now degrades to the deterministic scoring_service instead of raising. Also guards the response envelope itself (content[0].text). B3: single home for recommendation scoring services/recommendation_service.py holds the rules; routers/stats.py and backend/services/stats_service.py both delegate to it. Unified rules are the union of the two old copies: same weights/threshold, substring matching (superset of the old exact match), tolerant key aliases, health_flags rule kept. Endpoint response contract unchanged. Plus: Overpass circuit breaker and concurrent zone queries in geo_service - 128 sequential calls per analysis no longer each burn a connect timeout when the host has no outbound network. Tests: 152 -> 194 passed.
This commit is contained in:
+72
-8
@@ -1,9 +1,12 @@
|
||||
"""
|
||||
Geo service for building search zones and querying OpenStreetMap data via Overpass API.
|
||||
"""
|
||||
import asyncio
|
||||
import math
|
||||
import json
|
||||
import hashlib
|
||||
import logging
|
||||
import time
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import List, Dict, Optional, Tuple
|
||||
@@ -26,6 +29,52 @@ CACHE_DIR = Path("/tmp/overpass_cache")
|
||||
CACHE_TTL_HOURS = 24
|
||||
OVERPASS_URL = "https://overpass-api.de/api/interpreter"
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# --- Overpass circuit breaker ---------------------------------------------
|
||||
# build_search_zones issues ~128 Overpass calls per analysis. When the host has
|
||||
# no outbound connectivity every one of them burns the full connect timeout,
|
||||
# which turns a single analysis into several minutes of waiting for results
|
||||
# that are empty anyway. After OVERPASS_FAILURE_THRESHOLD consecutive
|
||||
# transport failures we stop calling out until OVERPASS_COOLDOWN_SECONDS have
|
||||
# passed. Callers get the same {'elements': []} they already got on error.
|
||||
OVERPASS_FAILURE_THRESHOLD = 3
|
||||
OVERPASS_COOLDOWN_SECONDS = 60.0
|
||||
OVERPASS_CONNECT_TIMEOUT = 5.0
|
||||
|
||||
_overpass_failures = 0
|
||||
_overpass_open_until = 0.0
|
||||
|
||||
|
||||
def _overpass_circuit_open() -> bool:
|
||||
"""True while the breaker is tripped (skip network, return empty fast)."""
|
||||
if _overpass_failures < OVERPASS_FAILURE_THRESHOLD:
|
||||
return False
|
||||
if time.monotonic() >= _overpass_open_until:
|
||||
_reset_overpass_circuit()
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def _record_overpass_failure() -> None:
|
||||
global _overpass_failures, _overpass_open_until
|
||||
_overpass_failures += 1
|
||||
if _overpass_failures == OVERPASS_FAILURE_THRESHOLD:
|
||||
_overpass_open_until = time.monotonic() + OVERPASS_COOLDOWN_SECONDS
|
||||
logger.warning(
|
||||
"Overpass API недоступен (%d подряд неудачных запросов). "
|
||||
"Геоданные отключены на %.0f c, анализ продолжается без них.",
|
||||
_overpass_failures,
|
||||
OVERPASS_COOLDOWN_SECONDS,
|
||||
)
|
||||
|
||||
|
||||
def _reset_overpass_circuit() -> None:
|
||||
global _overpass_failures, _overpass_open_until
|
||||
_overpass_failures = 0
|
||||
_overpass_open_until = 0.0
|
||||
|
||||
|
||||
# Direction mappings
|
||||
SEARCH_DISTANCES = [500, 1000, 2000, 5000]
|
||||
|
||||
@@ -167,8 +216,13 @@ async def query_overpass(query: str) -> Dict:
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
# Skip the network entirely while the breaker is tripped.
|
||||
if _overpass_circuit_open():
|
||||
return {'elements': []}
|
||||
|
||||
# Query API
|
||||
async with httpx.AsyncClient(timeout=30.0) as client:
|
||||
timeout = httpx.Timeout(30.0, connect=OVERPASS_CONNECT_TIMEOUT)
|
||||
async with httpx.AsyncClient(timeout=timeout) as client:
|
||||
try:
|
||||
response = await client.post(
|
||||
OVERPASS_URL,
|
||||
@@ -181,8 +235,10 @@ async def query_overpass(query: str) -> Dict:
|
||||
# Save to cache
|
||||
save_to_cache(cache_key, data)
|
||||
|
||||
_reset_overpass_circuit()
|
||||
return data
|
||||
except Exception as e:
|
||||
_record_overpass_failure()
|
||||
# Return empty result on error
|
||||
return {'elements': []}
|
||||
|
||||
@@ -296,8 +352,6 @@ async def get_zone_features(lat: float, lon: float, direction: str, radius_m: in
|
||||
);
|
||||
out geom;
|
||||
"""
|
||||
roads_data = await query_overpass(roads_query)
|
||||
roads_km = calculate_road_length(roads_data.get('elements', []))
|
||||
|
||||
# Query water bodies
|
||||
water_query = f"""
|
||||
@@ -308,8 +362,7 @@ async def get_zone_features(lat: float, lon: float, direction: str, radius_m: in
|
||||
);
|
||||
out center;
|
||||
"""
|
||||
water_data = await query_overpass(water_query)
|
||||
water_distance = find_nearest_distance(lat, lon, water_data.get('elements', []))
|
||||
|
||||
|
||||
# Query settlements
|
||||
settlement_query = f"""
|
||||
@@ -319,8 +372,7 @@ async def get_zone_features(lat: float, lon: float, direction: str, radius_m: in
|
||||
);
|
||||
out;
|
||||
"""
|
||||
settlement_data = await query_overpass(settlement_query)
|
||||
settlement_distance = find_nearest_distance(lat, lon, settlement_data.get('elements', []))
|
||||
|
||||
|
||||
# Query forests
|
||||
forest_query = f"""
|
||||
@@ -331,7 +383,19 @@ async def get_zone_features(lat: float, lon: float, direction: str, radius_m: in
|
||||
);
|
||||
out geom;
|
||||
"""
|
||||
forest_data = await query_overpass(forest_query)
|
||||
|
||||
|
||||
# The four queries are independent - issue them concurrently.
|
||||
roads_data, water_data, settlement_data, forest_data = await asyncio.gather(
|
||||
query_overpass(roads_query),
|
||||
query_overpass(water_query),
|
||||
query_overpass(settlement_query),
|
||||
query_overpass(forest_query),
|
||||
)
|
||||
|
||||
roads_km = calculate_road_length(roads_data.get('elements', []))
|
||||
water_distance = find_nearest_distance(lat, lon, water_data.get('elements', []))
|
||||
settlement_distance = find_nearest_distance(lat, lon, settlement_data.get('elements', []))
|
||||
forest_pct = calculate_forest_coverage(forest_data.get('elements', []), radius_m)
|
||||
|
||||
# Calculate road density (km of roads per km²)
|
||||
|
||||
Reference in New Issue
Block a user