build_communes_data.py 81 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264
  1. #!/usr/bin/env python3
  2. from __future__ import annotations
  3. import json
  4. import csv
  5. import time
  6. import difflib
  7. import math
  8. import unicodedata
  9. from concurrent.futures import ThreadPoolExecutor, as_completed
  10. from io import StringIO
  11. from pathlib import Path
  12. from typing import Iterable
  13. from urllib.parse import urlencode
  14. from urllib.request import Request, urlopen
  15. ROOT_DIR = Path(__file__).resolve().parents[1]
  16. DATA_DIR = ROOT_DIR / "data"
  17. TERRITORY_DIR = DATA_DIR / "territories"
  18. API_BASE_URL = "https://geo.api.gouv.fr"
  19. REGIONS_GEOJSON_URL = (
  20. "https://raw.githubusercontent.com/gregoiredavid/france-geojson/master/"
  21. "regions-version-simplifiee.geojson"
  22. )
  23. DEPARTMENTS_GEOJSON_URL = (
  24. "https://raw.githubusercontent.com/gregoiredavid/france-geojson/master/"
  25. "departements-version-simplifiee.geojson"
  26. )
  27. COASTLINE_GEOJSON_URL = (
  28. "https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/geojson/"
  29. "ne_50m_coastline.geojson"
  30. )
  31. REGION_COMMUNES_GEOJSON_BASE_URL = (
  32. "https://raw.githubusercontent.com/gregoiredavid/france-geojson/master/regions"
  33. )
  34. COMMUNE_TERRITORIAL_COMPETENCE_URL = (
  35. "https://www.data.gouv.fr/api/1/datasets/r/c53cd4d4-4623-4772-9b8c-bc72a9cdf4c2"
  36. )
  37. GENDARMERIE_PUBLIC_UNITS_URL = (
  38. "https://www.data.gouv.fr/api/1/datasets/r/17320fe6-a896-4686-93e6-502be2ad23f2"
  39. )
  40. SERVICE_PUBLIC_ANNUAIRE_API_URL = (
  41. "https://api-lannuaire.service-public.gouv.fr/api/explore/v2.1/catalog/datasets/"
  42. "api-lannuaire-administration/records"
  43. )
  44. REQUEST_HEADERS = {
  45. "User-Agent": "GiePlaces data builder",
  46. "Accept": "application/json",
  47. }
  48. COASTLINE_GRID_SIZE = 1.0
  49. COASTLINE_DISTANCE_THRESHOLD = 0.08
  50. REGION_ENTITY_TYPE = "region"
  51. DEPARTMENT_ENTITY_TYPE = "departement"
  52. COMMUNE_ENTITY_TYPE = "commune"
  53. OVERSEAS_TERRITORIES = [
  54. {"code": "971", "label": "Guadeloupe", "shortLabel": "Guadeloupe"},
  55. {"code": "972", "label": "Martinique", "shortLabel": "Martinique"},
  56. {"code": "973", "label": "Guyane", "shortLabel": "Guyane"},
  57. {"code": "974", "label": "La Réunion", "shortLabel": "La Réunion"},
  58. {"code": "976", "label": "Mayotte", "shortLabel": "Mayotte"},
  59. {"code": "975", "label": "Saint-Pierre-et-Miquelon", "shortLabel": "SPM"},
  60. {"code": "977", "label": "Saint-Barthélemy", "shortLabel": "Saint-Barth"},
  61. {"code": "978", "label": "Saint-Martin", "shortLabel": "Saint-Martin"},
  62. {"code": "986", "label": "Wallis-et-Futuna", "shortLabel": "Wallis"},
  63. {"code": "987", "label": "Polynésie française", "shortLabel": "Polynésie"},
  64. {"code": "988", "label": "Nouvelle-Calédonie", "shortLabel": "Nouvelle-Calédonie"},
  65. ]
  66. POLYNESIE_RETAINED_COMMUNES = {
  67. "Arue",
  68. "Faaa",
  69. "Hitiaa O Te Ra",
  70. "Mahina",
  71. "Moorea-Maiao",
  72. "Paea",
  73. "Papara",
  74. "Papeete",
  75. "Pirae",
  76. "Punaauia",
  77. "Taiarapu-Est",
  78. "Taiarapu-Ouest",
  79. "Teva I Uta",
  80. }
  81. POLYNESIE_TRIMMED_COMMUNES = {
  82. "Taiarapu-Est",
  83. "Moorea-Maiao",
  84. "Arue",
  85. }
  86. SPECIAL_TERRITORY_SIMPLIFICATION = {
  87. "polynesie-francaise": {
  88. "minDistance": 0.0018,
  89. "minArea": 0.0000018,
  90. },
  91. "nouvelle-caledonie": {
  92. "minDistance": 0.0014,
  93. "minArea": 0.0000012,
  94. },
  95. }
  96. SPECIAL_TERRITORY_LAYOUTS = {
  97. "polynesie-francaise": {
  98. "marquises": {"target": (-151.2, -16.4), "scale": 1.0},
  99. "societe": {"target": (-150.8, -16.8), "scale": 1.0},
  100. "tuamotu-ouest": {"target": (-150.4, -16.8), "scale": 1.0},
  101. "tuamotu-centre": {"target": (-150.0, -16.8), "scale": 1.0},
  102. "tuamotu-est": {"target": (-149.6, -16.8), "scale": 1.0},
  103. "australes": {"target": (-150.2, -17.2), "scale": 1.0},
  104. "gambier": {"target": (-149.8, -17.2), "scale": 1.0},
  105. },
  106. "wallis-et-futuna": {
  107. "uvea": {"target": (-176.22, -13.32), "scale": 1.0},
  108. "futuna": {"target": (-176.68, -13.88), "scale": 1.0},
  109. },
  110. }
  111. SPECIAL_PLANAR_LAYOUT_ROWS = {}
  112. PLANAR_TERRITORY_COMPACTION = {
  113. "polynesie-francaise": {
  114. "factor": 1.0,
  115. },
  116. "nouvelle-caledonie": {
  117. "factor": 0.6,
  118. },
  119. }
  120. SPECIAL_TERRITORY_QUANTIZATION = {
  121. "polynesie-francaise": 4,
  122. "nouvelle-caledonie": 4,
  123. }
  124. PREPROJECTED_TERRITORIES = {
  125. "polynesie-francaise",
  126. "nouvelle-caledonie",
  127. }
  128. SPECIAL_GROUP_DISTANCE_REDUCTION = {}
  129. SPECIAL_GROUP_MIN_GAP_FACTOR = {
  130. "polynesie-francaise": 0.0005,
  131. }
  132. SPECIAL_GROUP_PACK_BY_FEATURE = {
  133. "polynesie-francaise": True,
  134. }
  135. SPECIAL_GROUP_EFFECTIVE_BOUNDS_FACTOR = {
  136. "polynesie-francaise": 0.18,
  137. }
  138. DAY_LABELS = [
  139. ("lundi", "Lun"),
  140. ("mardi", "Mar"),
  141. ("mercredi", "Mer"),
  142. ("jeudi", "Jeu"),
  143. ("vendredi", "Ven"),
  144. ("samedi", "Sam"),
  145. ("dimanche", "Dim"),
  146. ("jours_feries", "Fériés"),
  147. ]
  148. def fetch_json(url: str, retries: int = 3) -> dict | list:
  149. last_error = None
  150. for attempt in range(1, retries + 1):
  151. try:
  152. request = Request(url, headers=REQUEST_HEADERS)
  153. with urlopen(request, timeout=60) as response:
  154. return json.load(response)
  155. except Exception as exc: # pragma: no cover - network failures are not deterministic
  156. last_error = exc
  157. if attempt == retries:
  158. raise
  159. time.sleep(attempt * 0.75)
  160. raise RuntimeError(f"Unable to fetch {url}: {last_error}")
  161. def fetch_text(url: str, retries: int = 3) -> str:
  162. last_error = None
  163. for attempt in range(1, retries + 1):
  164. try:
  165. request = Request(url, headers=REQUEST_HEADERS)
  166. with urlopen(request, timeout=60) as response:
  167. return response.read().decode("utf-8-sig")
  168. except Exception as exc: # pragma: no cover - network failures are not deterministic
  169. last_error = exc
  170. if attempt == retries:
  171. raise
  172. time.sleep(attempt * 0.75)
  173. raise RuntimeError(f"Unable to fetch {url}: {last_error}")
  174. def slugify(value: str) -> str:
  175. normalized = unicodedata.normalize("NFD", str(value))
  176. ascii_value = "".join(char for char in normalized if unicodedata.category(char) != "Mn")
  177. chunks = []
  178. for char in ascii_value.lower():
  179. chunks.append(char if char.isalnum() else "-")
  180. slug = "".join(chunks)
  181. while "--" in slug:
  182. slug = slug.replace("--", "-")
  183. return slug.strip("-")
  184. def normalize_sort_text(value: str) -> str:
  185. normalized = unicodedata.normalize("NFD", str(value))
  186. ascii_value = "".join(char for char in normalized if unicodedata.category(char) != "Mn")
  187. lowered = ascii_value.lower()
  188. cleaned = []
  189. for char in lowered:
  190. cleaned.append(char if char.isalnum() else " ")
  191. return " ".join("".join(cleaned).split())
  192. def department_territory_key(department_code: str) -> str:
  193. return f"departement-{slugify(department_code)}"
  194. def region_territory_key(region_name: str) -> str:
  195. return f"region-{slugify(region_name)}"
  196. def get_department_metadata() -> list[dict]:
  197. url = f"{API_BASE_URL}/departements?fields=code,nom,codeRegion,zone&format=json"
  198. return fetch_json(url)
  199. def get_region_metadata() -> list[dict]:
  200. url = f"{API_BASE_URL}/regions?fields=code,nom&format=json"
  201. return fetch_json(url)
  202. def get_commune_postal_codes_by_code() -> dict[str, list[str]]:
  203. url = f"{API_BASE_URL}/communes?fields=code,codesPostaux&format=json"
  204. payload = fetch_json(url)
  205. return {
  206. item["code"]: sorted({str(postal_code) for postal_code in item.get("codesPostaux", []) if postal_code})
  207. for item in payload
  208. }
  209. def build_service_key(institution: str, service_id: str, service_label: str, commune_code: str) -> str:
  210. normalized_institution = (institution or "UNK").upper()
  211. normalized_service_id = str(service_id or "").strip()
  212. normalized_service_label = str(service_label or "").strip()
  213. if normalized_institution == "GN" and normalized_service_id:
  214. return f"gn:{normalized_service_id}"
  215. if normalized_institution == "PN" and normalized_service_label:
  216. return f"pn:{slugify(normalized_service_label)}"
  217. if normalized_service_label:
  218. return f"{slugify(normalized_institution)}:{slugify(normalized_service_label)}"
  219. return f"unk:{commune_code}"
  220. def normalize_unit_match_text(value: str) -> str:
  221. normalized = unicodedata.normalize("NFD", str(value))
  222. ascii_value = "".join(char for char in normalized if unicodedata.category(char) != "Mn")
  223. lowered = ascii_value.lower().replace("'", " ")
  224. cleaned = []
  225. for char in lowered:
  226. cleaned.append(char if char.isalnum() else " ")
  227. return " ".join("".join(cleaned).split())
  228. def get_label_similarity(left: str, right: str) -> float:
  229. return difflib.SequenceMatcher(None, normalize_unit_match_text(left), normalize_unit_match_text(right)).ratio()
  230. def normalize_service_label(institution: str, service_label: str) -> str:
  231. normalized_label = str(service_label or "").strip()
  232. normalized_institution = (institution or "").upper()
  233. if normalized_label:
  234. return normalized_label
  235. if normalized_institution == "PN":
  236. return "Police nationale"
  237. if normalized_institution == "GN":
  238. return "Gendarmerie nationale"
  239. return "Compétence territoriale non renseignée"
  240. def get_commune_territorial_competence_data() -> tuple[dict[str, dict], dict[str, dict], dict[str, set[str]]]:
  241. payload = fetch_text(COMMUNE_TERRITORIAL_COMPETENCE_URL)
  242. reader = csv.DictReader(StringIO(payload), delimiter=";")
  243. rows_by_commune_code: dict[str, list[dict]] = {}
  244. service_registry: dict[str, dict] = {}
  245. commune_codes_by_service_key: dict[str, set[str]] = {}
  246. for row in reader:
  247. commune_code = str(row.get("code_commune", "")).strip()
  248. if not commune_code:
  249. continue
  250. institution = str(row.get("institution", "")).strip().upper()
  251. service_id = str(row.get("id_service", "")).strip()
  252. service_label = normalize_service_label(institution, row.get("service", ""))
  253. service_key = build_service_key(institution, service_id, service_label, commune_code)
  254. service_registry[service_key] = {
  255. "i": institution or "UNK",
  256. "l": service_label,
  257. }
  258. commune_codes_by_service_key.setdefault(service_key, set()).add(commune_code)
  259. rows_by_commune_code.setdefault(commune_code, []).append(
  260. {
  261. "institution": institution or "UNK",
  262. "serviceKey": service_key,
  263. "serviceLabel": service_label,
  264. }
  265. )
  266. normalized_competence_by_commune_code: dict[str, dict] = {}
  267. for commune_code, rows in rows_by_commune_code.items():
  268. unique_rows = []
  269. seen_rows = set()
  270. for row in rows:
  271. row_key = (row["institution"], row["serviceKey"], row["serviceLabel"])
  272. if row_key in seen_rows:
  273. continue
  274. seen_rows.add(row_key)
  275. unique_rows.append(row)
  276. if len(unique_rows) == 1:
  277. normalized_competence_by_commune_code[commune_code] = {
  278. "i": unique_rows[0]["institution"],
  279. "u": unique_rows[0]["serviceKey"],
  280. }
  281. continue
  282. institutions = {row["institution"] for row in unique_rows}
  283. if "GN" in institutions and "PN" in institutions:
  284. institution = "MX"
  285. elif "PN" in institutions:
  286. institution = "PN"
  287. elif "GN" in institutions:
  288. institution = "GN"
  289. else:
  290. institution = "UNK"
  291. normalized_competence_by_commune_code[commune_code] = {
  292. "i": institution,
  293. "m": " / ".join(row["serviceLabel"] for row in unique_rows),
  294. "v": [row["serviceKey"] for row in unique_rows],
  295. }
  296. return normalized_competence_by_commune_code, service_registry, commune_codes_by_service_key
  297. def format_public_unit_hours(row: dict) -> str:
  298. day_chunks = []
  299. for day_key, day_label in DAY_LABELS:
  300. ranges = []
  301. for slot in range(1, 4):
  302. start = str(row.get(f"{day_key}_plage{slot}_debut", "")).strip()
  303. end = str(row.get(f"{day_key}_plage{slot}_fin", "")).strip()
  304. if start and end:
  305. ranges.append(f"{start}-{end}")
  306. if ranges:
  307. day_chunks.append(f"{day_label} {' / '.join(ranges)}")
  308. return "; ".join(day_chunks)
  309. def parse_annuaire_sequence(value: object) -> list[dict]:
  310. if isinstance(value, list):
  311. return [item for item in value if isinstance(item, dict)]
  312. if isinstance(value, str):
  313. raw_value = value.strip()
  314. if not raw_value:
  315. return []
  316. try:
  317. parsed = json.loads(raw_value)
  318. except json.JSONDecodeError:
  319. return []
  320. if isinstance(parsed, list):
  321. return [item for item in parsed if isinstance(item, dict)]
  322. return []
  323. def fetch_annuaire_records(where: str) -> list[dict]:
  324. results = []
  325. offset = 0
  326. limit = 100
  327. while True:
  328. query = urlencode({"where": where, "limit": limit, "offset": offset})
  329. payload = fetch_json(f"{SERVICE_PUBLIC_ANNUAIRE_API_URL}?{query}")
  330. batch = payload.get("results", [])
  331. if not batch:
  332. break
  333. results.extend(batch)
  334. offset += len(batch)
  335. if offset >= payload.get("total_count", 0):
  336. break
  337. return results
  338. def format_annuaire_hours(opening_ranges: object) -> str:
  339. parsed_ranges = parse_annuaire_sequence(opening_ranges)
  340. day_chunks = []
  341. for entry in parsed_ranges:
  342. start_day = str(entry.get("nom_jour_debut", "")).strip()
  343. end_day = str(entry.get("nom_jour_fin", "")).strip()
  344. day_label = start_day if not end_day or end_day == start_day else f"{start_day}-{end_day}"
  345. ranges = []
  346. for slot in range(1, 3):
  347. start = str(entry.get(f"valeur_heure_debut_{slot}", "")).strip()
  348. end = str(entry.get(f"valeur_heure_fin_{slot}", "")).strip()
  349. if start and end:
  350. ranges.append(f"{start[:5]}-{end[:5]}")
  351. if day_label and ranges:
  352. day_chunks.append(f"{day_label} {' / '.join(ranges)}")
  353. return "; ".join(day_chunks)
  354. def get_annuaire_address_value(addresses: object) -> str:
  355. parsed_addresses = parse_annuaire_sequence(addresses)
  356. if not parsed_addresses:
  357. return ""
  358. address = parsed_addresses[0]
  359. chunks = [
  360. str(address.get("numero_voie", "")).strip(),
  361. str(address.get("complement1", "")).strip(),
  362. str(address.get("complement2", "")).strip(),
  363. str(address.get("service_distribution", "")).strip(),
  364. str(address.get("code_postal", "")).strip(),
  365. str(address.get("nom_commune", "")).strip(),
  366. ]
  367. return " ".join(chunk for chunk in chunks if chunk)
  368. def get_annuaire_phone_value(phones: object) -> str:
  369. parsed_phones = parse_annuaire_sequence(phones)
  370. if not parsed_phones:
  371. return ""
  372. return str(parsed_phones[0].get("valeur", "")).strip()
  373. def get_gendarmerie_public_units_by_service_key(service_registry: dict[str, dict]) -> dict[str, dict]:
  374. payload = fetch_text(GENDARMERIE_PUBLIC_UNITS_URL)
  375. reader = csv.DictReader(StringIO(payload), delimiter=";")
  376. units_by_service_key: dict[str, dict] = {}
  377. for row in reader:
  378. service_id = str(row.get("identifiant_public_unite", "")).strip()
  379. if not service_id:
  380. continue
  381. service_key = f"gn:{service_id}"
  382. service = service_registry.get(service_key)
  383. if not service or service.get("i") != "GN":
  384. continue
  385. longitude = str(row.get("geocodage_x_GPS", "")).strip()
  386. latitude = str(row.get("geocodage_y_GPS", "")).strip()
  387. if not longitude or not latitude:
  388. continue
  389. units_by_service_key[service_key] = {
  390. "s": service_key,
  391. "i": "GN",
  392. "n": service.get("l") or str(row.get("service", "")).strip() or f"Unité {service_id}",
  393. "x": round(float(longitude), 5),
  394. "y": round(float(latitude), 5),
  395. "c": str(row.get("code_commune_insee", "")).strip(),
  396. "m": str(row.get("commune", "")).strip(),
  397. "a": str(row.get("adresse_geographique", "")).strip(),
  398. "t": str(row.get("telephone", "")).strip(),
  399. "h": format_public_unit_hours(row),
  400. "u": str(row.get("url", "")).strip(),
  401. }
  402. return units_by_service_key
  403. def get_police_public_units_by_service_key(
  404. service_registry: dict[str, dict],
  405. commune_codes_by_service_key: dict[str, set[str]],
  406. ) -> dict[str, dict]:
  407. source_rows = fetch_annuaire_records("partenaire = 'commissariat'")
  408. annuaire_records = [
  409. {
  410. "serviceKey": build_service_key("PN", "", row.get("nom", ""), ""),
  411. "name": str(row.get("nom", "")).strip(),
  412. "communeCode": str(row.get("code_insee_commune", "")).strip(),
  413. "hostCommune": str((parse_annuaire_sequence(row.get("adresse")) or [{}])[0].get("nom_commune", "")).strip(),
  414. "address": get_annuaire_address_value(row.get("adresse")),
  415. "phone": get_annuaire_phone_value(row.get("telephone")),
  416. "hours": format_annuaire_hours(row.get("plage_ouverture")),
  417. "url": str(row.get("url_service_public", "")).strip(),
  418. "longitude": str((parse_annuaire_sequence(row.get("adresse")) or [{}])[0].get("longitude", "")).strip(),
  419. "latitude": str((parse_annuaire_sequence(row.get("adresse")) or [{}])[0].get("latitude", "")).strip(),
  420. }
  421. for row in source_rows
  422. ]
  423. units_by_service_key: dict[str, dict] = {}
  424. police_service_keys = [
  425. service_key
  426. for service_key, service in service_registry.items()
  427. if service.get("i") == "PN"
  428. ]
  429. for service_key in police_service_keys:
  430. service_label = service_registry[service_key]["l"]
  431. commune_codes = commune_codes_by_service_key.get(service_key, set())
  432. commune_candidates = [
  433. record
  434. for record in annuaire_records
  435. if record["communeCode"] and record["communeCode"] in commune_codes
  436. ]
  437. best_match = None
  438. best_ratio = 0.0
  439. for record in commune_candidates or annuaire_records:
  440. similarity = get_label_similarity(service_label, record["name"])
  441. if similarity > best_ratio:
  442. best_ratio = similarity
  443. best_match = record
  444. if not best_match:
  445. continue
  446. minimum_ratio = 0.45 if commune_candidates else 0.84
  447. if best_ratio < minimum_ratio:
  448. continue
  449. if not best_match["longitude"] or not best_match["latitude"]:
  450. continue
  451. units_by_service_key[service_key] = {
  452. "s": service_key,
  453. "i": "PN",
  454. "n": service_label,
  455. "x": round(float(best_match["longitude"]), 5),
  456. "y": round(float(best_match["latitude"]), 5),
  457. "c": best_match["communeCode"],
  458. "m": best_match["hostCommune"],
  459. "a": best_match["address"],
  460. "t": best_match["phone"],
  461. "h": best_match["hours"],
  462. "u": best_match["url"],
  463. }
  464. return units_by_service_key
  465. def get_department_code_from_commune_code(commune_code: str) -> str:
  466. code = str(commune_code)
  467. if code.startswith("2A") or code.startswith("2B"):
  468. return code[:2]
  469. if code.startswith("97") or code.startswith("98"):
  470. return code[:3]
  471. return code[:2]
  472. def quantize_pair(pair: Iterable[float], precision: int = 5) -> list[float]:
  473. return [round(float(pair[0]), precision), round(float(pair[1]), precision)]
  474. def quantize_geometry(geometry: dict, precision: int = 5) -> dict:
  475. geometry_type = geometry.get("type")
  476. coordinates = geometry.get("coordinates")
  477. if geometry_type == "Polygon":
  478. return {
  479. "type": "Polygon",
  480. "coordinates": [
  481. [quantize_pair(point, precision) for point in ring]
  482. for ring in coordinates
  483. ],
  484. }
  485. if geometry_type == "MultiPolygon":
  486. return {
  487. "type": "MultiPolygon",
  488. "coordinates": [
  489. [
  490. [quantize_pair(point, precision) for point in ring]
  491. for ring in polygon
  492. ]
  493. for polygon in coordinates
  494. ],
  495. }
  496. raise ValueError(f"Unsupported geometry type: {geometry_type}")
  497. def iter_geometry_points(geometry: dict) -> Iterable[list[float]]:
  498. geometry_type = geometry.get("type")
  499. coordinates = geometry.get("coordinates", [])
  500. if geometry_type == "Polygon":
  501. for ring in coordinates:
  502. for point in ring:
  503. yield point
  504. return
  505. if geometry_type == "MultiPolygon":
  506. for polygon in coordinates:
  507. for ring in polygon:
  508. for point in ring:
  509. yield point
  510. return
  511. raise ValueError(f"Unsupported geometry type: {geometry_type}")
  512. def get_geometry_center(geometry: dict) -> tuple[float, float]:
  513. min_longitude = float("inf")
  514. min_latitude = float("inf")
  515. max_longitude = float("-inf")
  516. max_latitude = float("-inf")
  517. for longitude, latitude in iter_geometry_points(geometry):
  518. min_longitude = min(min_longitude, longitude)
  519. min_latitude = min(min_latitude, latitude)
  520. max_longitude = max(max_longitude, longitude)
  521. max_latitude = max(max_latitude, latitude)
  522. return (
  523. (min_longitude + max_longitude) / 2,
  524. (min_latitude + max_latitude) / 2,
  525. )
  526. def simplify_ring(ring: list[list[float]], min_distance: float, min_area: float) -> list[list[float]]:
  527. if len(ring) < 4:
  528. return ring
  529. is_closed = ring[0] == ring[-1]
  530. points = ring[:-1] if is_closed else ring[:]
  531. if len(points) < 3:
  532. return ring
  533. filtered = [points[0]]
  534. min_distance_sq = min_distance * min_distance
  535. for point in points[1:]:
  536. delta_longitude = point[0] - filtered[-1][0]
  537. delta_latitude = point[1] - filtered[-1][1]
  538. if delta_longitude * delta_longitude + delta_latitude * delta_latitude >= min_distance_sq:
  539. filtered.append(point)
  540. if len(filtered) < 3:
  541. filtered = points[:]
  542. changed = True
  543. while changed and len(filtered) > 3:
  544. changed = False
  545. reduced = [filtered[0]]
  546. for index in range(1, len(filtered) - 1):
  547. previous_point = reduced[-1]
  548. point = filtered[index]
  549. next_point = filtered[index + 1]
  550. cross_product = abs(
  551. (point[0] - previous_point[0]) * (next_point[1] - previous_point[1])
  552. - (point[1] - previous_point[1]) * (next_point[0] - previous_point[0])
  553. )
  554. if cross_product <= min_area:
  555. changed = True
  556. continue
  557. reduced.append(point)
  558. reduced.append(filtered[-1])
  559. filtered = reduced
  560. if is_closed:
  561. filtered.append(filtered[0])
  562. return filtered
  563. def simplify_geometry(geometry: dict, min_distance: float, min_area: float) -> dict:
  564. geometry_type = geometry.get("type")
  565. coordinates = geometry.get("coordinates", [])
  566. if geometry_type == "Polygon":
  567. return {
  568. "type": "Polygon",
  569. "coordinates": [simplify_ring(ring, min_distance, min_area) for ring in coordinates],
  570. }
  571. if geometry_type == "MultiPolygon":
  572. return {
  573. "type": "MultiPolygon",
  574. "coordinates": [
  575. [simplify_ring(ring, min_distance, min_area) for ring in polygon]
  576. for polygon in coordinates
  577. ],
  578. }
  579. raise ValueError(f"Unsupported geometry type: {geometry_type}")
  580. def get_ring_area(ring: list[list[float]]) -> float:
  581. if len(ring) < 3:
  582. return 0.0
  583. area = 0.0
  584. previous_x, previous_y = ring[-1]
  585. for point_x, point_y in ring:
  586. area += previous_x * point_y - point_x * previous_y
  587. previous_x, previous_y = point_x, point_y
  588. return abs(area) / 2
  589. def get_polygon_area(polygon: list[list[list[float]]]) -> float:
  590. if not polygon:
  591. return 0.0
  592. outer_area = get_ring_area(polygon[0])
  593. inner_area = sum(get_ring_area(ring) for ring in polygon[1:])
  594. return max(0.0, outer_area - inner_area)
  595. def get_polygon_center(polygon: list[list[list[float]]]) -> tuple[float, float]:
  596. min_x = math.inf
  597. min_y = math.inf
  598. max_x = -math.inf
  599. max_y = -math.inf
  600. for ring in polygon:
  601. for point_x, point_y in ring:
  602. min_x = min(min_x, point_x)
  603. min_y = min(min_y, point_y)
  604. max_x = max(max_x, point_x)
  605. max_y = max(max_y, point_y)
  606. return (min_x + max_x) / 2, (min_y + max_y) / 2
  607. def trim_special_commune_polygons(territory_key: str, commune_name: str, geometry: dict) -> dict:
  608. if (
  609. territory_key != "polynesie-francaise"
  610. or commune_name not in POLYNESIE_TRIMMED_COMMUNES
  611. or geometry.get("type") != "MultiPolygon"
  612. ):
  613. return geometry
  614. polygons = geometry.get("coordinates", [])
  615. if len(polygons) <= 1:
  616. return geometry
  617. polygon_areas = [get_polygon_area(polygon) for polygon in polygons]
  618. main_index = max(range(len(polygons)), key=lambda index: polygon_areas[index])
  619. main_polygon = polygons[main_index]
  620. return {
  621. "type": "MultiPolygon",
  622. "coordinates": [main_polygon],
  623. }
  624. def get_special_layout_group_id(territory_key: str, longitude: float, latitude: float) -> str | None:
  625. if territory_key == "wallis-et-futuna":
  626. return "uvea" if longitude > -177.0 else "futuna"
  627. if territory_key == "polynesie-francaise":
  628. if latitude > -12.0:
  629. return "marquises"
  630. if latitude <= -21.2:
  631. return "gambier" if longitude >= -140.5 else "australes"
  632. if longitude < -148.5:
  633. return "societe"
  634. if longitude < -145.8:
  635. return "tuamotu-ouest"
  636. if longitude < -143.5:
  637. return "tuamotu-centre"
  638. return "tuamotu-est"
  639. return None
  640. def get_layout_source_centers(territory_key: str, features: list[dict]) -> dict[str, tuple[float, float]]:
  641. layout_config = SPECIAL_TERRITORY_LAYOUTS.get(territory_key)
  642. if not layout_config:
  643. return {}
  644. grouped_centers: dict[str, list[tuple[float, float]]] = {}
  645. for feature in features:
  646. center_longitude, center_latitude = get_geometry_center(feature["geometry"])
  647. group_id = get_special_layout_group_id(territory_key, center_longitude, center_latitude)
  648. if not group_id:
  649. continue
  650. grouped_centers.setdefault(group_id, []).append((center_longitude, center_latitude))
  651. source_centers = {}
  652. for group_id, centers in grouped_centers.items():
  653. source_centers[group_id] = (
  654. sum(center[0] for center in centers) / len(centers),
  655. sum(center[1] for center in centers) / len(centers),
  656. )
  657. return source_centers
  658. def transform_point_for_layout(
  659. territory_key: str,
  660. longitude: float,
  661. latitude: float,
  662. source_centers: dict[str, tuple[float, float]],
  663. ) -> list[float]:
  664. group_id = get_special_layout_group_id(territory_key, longitude, latitude)
  665. layout_config = SPECIAL_TERRITORY_LAYOUTS.get(territory_key, {})
  666. group_layout = layout_config.get(group_id or "")
  667. source_center = source_centers.get(group_id or "")
  668. if not group_layout or not source_center:
  669. return [round(float(longitude), 5), round(float(latitude), 5)]
  670. target_longitude, target_latitude = group_layout["target"]
  671. scale = group_layout["scale"]
  672. return [
  673. round(target_longitude + scale * (longitude - source_center[0]), 5),
  674. round(target_latitude + scale * (latitude - source_center[1]), 5),
  675. ]
  676. def transform_geometry_for_layout(
  677. territory_key: str,
  678. geometry: dict,
  679. source_centers: dict[str, tuple[float, float]],
  680. ) -> dict:
  681. geometry_type = geometry.get("type")
  682. coordinates = geometry.get("coordinates", [])
  683. if geometry_type == "Polygon":
  684. return {
  685. "type": "Polygon",
  686. "coordinates": [
  687. [
  688. transform_point_for_layout(territory_key, point[0], point[1], source_centers)
  689. for point in ring
  690. ]
  691. for ring in coordinates
  692. ],
  693. }
  694. if geometry_type == "MultiPolygon":
  695. return {
  696. "type": "MultiPolygon",
  697. "coordinates": [
  698. [
  699. [
  700. transform_point_for_layout(territory_key, point[0], point[1], source_centers)
  701. for point in ring
  702. ]
  703. for ring in polygon
  704. ]
  705. for polygon in coordinates
  706. ],
  707. }
  708. raise ValueError(f"Unsupported geometry type: {geometry_type}")
  709. def project_point_to_planar_mercator(point: list[float]) -> list[float]:
  710. longitude = math.radians(float(point[0]))
  711. latitude = math.radians(float(point[1]))
  712. clamped_latitude = max(min(latitude, math.radians(89.9999)), math.radians(-89.9999))
  713. mercator_y = math.log(math.tan(math.pi / 4 + clamped_latitude / 2))
  714. return [round(longitude, 6), round(-mercator_y, 6)]
  715. def project_geometry_to_planar_mercator(geometry: dict) -> dict:
  716. geometry_type = geometry.get("type")
  717. coordinates = geometry.get("coordinates", [])
  718. if geometry_type == "Polygon":
  719. return {
  720. "type": "Polygon",
  721. "coordinates": [
  722. [project_point_to_planar_mercator(point) for point in ring]
  723. for ring in coordinates
  724. ],
  725. }
  726. if geometry_type == "MultiPolygon":
  727. return {
  728. "type": "MultiPolygon",
  729. "coordinates": [
  730. [
  731. [project_point_to_planar_mercator(point) for point in ring]
  732. for ring in polygon
  733. ]
  734. for polygon in coordinates
  735. ],
  736. }
  737. raise ValueError(f"Unsupported geometry type: {geometry_type}")
  738. def project_units_to_planar_mercator(units: list[dict]) -> list[dict]:
  739. projected_units = []
  740. for unit in units:
  741. projected_longitude, projected_latitude = project_point_to_planar_mercator([unit["x"], unit["y"]])
  742. projected_unit = dict(unit)
  743. projected_unit["x"] = projected_longitude
  744. projected_unit["y"] = projected_latitude
  745. projected_units.append(projected_unit)
  746. return projected_units
  747. def compact_planar_point(point_x: float, point_y: float, origin_x: float, origin_y: float, compact_factor: float) -> list[float]:
  748. return [
  749. round(origin_x + (point_x - origin_x) * compact_factor, 6),
  750. round(origin_y + (point_y - origin_y) * compact_factor, 6),
  751. ]
  752. def compact_planar_geometry(
  753. geometry: dict,
  754. origin_x: float,
  755. origin_y: float,
  756. compact_factor: float,
  757. ) -> dict:
  758. geometry_type = geometry.get("type")
  759. coordinates = geometry.get("coordinates", [])
  760. if geometry_type == "Polygon":
  761. return {
  762. "type": "Polygon",
  763. "coordinates": [
  764. [compact_planar_point(point_x, point_y, origin_x, origin_y, compact_factor) for point_x, point_y in ring]
  765. for ring in coordinates
  766. ],
  767. }
  768. if geometry_type == "MultiPolygon":
  769. return {
  770. "type": "MultiPolygon",
  771. "coordinates": [
  772. [
  773. [
  774. compact_planar_point(point_x, point_y, origin_x, origin_y, compact_factor)
  775. for point_x, point_y in ring
  776. ]
  777. for ring in polygon
  778. ]
  779. for polygon in coordinates
  780. ],
  781. }
  782. raise ValueError(f"Unsupported geometry type: {geometry_type}")
  783. def compact_planar_features(
  784. projected_features: list[dict],
  785. compact_factor: float,
  786. ) -> tuple[list[dict], tuple[float, float] | None]:
  787. if compact_factor >= 1 or not projected_features:
  788. return projected_features, None
  789. bounds_min_x = math.inf
  790. bounds_min_y = math.inf
  791. bounds_max_x = -math.inf
  792. bounds_max_y = -math.inf
  793. for feature in projected_features:
  794. min_x, min_y, max_x, max_y = get_geometry_bounds(feature["geometry"])
  795. bounds_min_x = min(bounds_min_x, min_x)
  796. bounds_min_y = min(bounds_min_y, min_y)
  797. bounds_max_x = max(bounds_max_x, max_x)
  798. bounds_max_y = max(bounds_max_y, max_y)
  799. if not math.isfinite(bounds_min_x) or not math.isfinite(bounds_max_x):
  800. return projected_features, None
  801. origin_x = (bounds_min_x + bounds_max_x) / 2
  802. origin_y = (bounds_min_y + bounds_max_y) / 2
  803. compacted_features = [
  804. {
  805. "type": "Feature",
  806. "properties": dict(feature["properties"]),
  807. "geometry": compact_planar_geometry(feature["geometry"], origin_x, origin_y, compact_factor),
  808. }
  809. for feature in projected_features
  810. ]
  811. return compacted_features, (origin_x, origin_y)
  812. def compact_planar_units(units: list[dict], origin_x: float, origin_y: float, compact_factor: float) -> list[dict]:
  813. if compact_factor >= 1 or not units:
  814. return units
  815. compacted_units = []
  816. for unit in units:
  817. compact_unit = dict(unit)
  818. compact_unit["x"] = round(origin_x + (float(unit["x"]) - origin_x) * compact_factor, 6)
  819. compact_unit["y"] = round(origin_y + (float(unit["y"]) - origin_y) * compact_factor, 6)
  820. compacted_units.append(compact_unit)
  821. return compacted_units
  822. def get_geometry_bounds(geometry: dict) -> tuple[float, float, float, float]:
  823. min_x = math.inf
  824. min_y = math.inf
  825. max_x = -math.inf
  826. max_y = -math.inf
  827. coordinates = geometry.get("coordinates", [])
  828. polygons = coordinates if geometry.get("type") == "MultiPolygon" else [coordinates]
  829. for polygon in polygons:
  830. for ring in polygon:
  831. for point_x, point_y in ring:
  832. min_x = min(min_x, point_x)
  833. min_y = min(min_y, point_y)
  834. max_x = max(max_x, point_x)
  835. max_y = max(max_y, point_y)
  836. return min_x, min_y, max_x, max_y
  837. def translate_geometry(geometry: dict, offset_x: float, offset_y: float) -> dict:
  838. geometry_type = geometry.get("type")
  839. coordinates = geometry.get("coordinates", [])
  840. if geometry_type == "Polygon":
  841. return {
  842. "type": "Polygon",
  843. "coordinates": [
  844. [[round(point_x + offset_x, 6), round(point_y + offset_y, 6)] for point_x, point_y in ring]
  845. for ring in coordinates
  846. ],
  847. }
  848. if geometry_type == "MultiPolygon":
  849. return {
  850. "type": "MultiPolygon",
  851. "coordinates": [
  852. [
  853. [[round(point_x + offset_x, 6), round(point_y + offset_y, 6)] for point_x, point_y in ring]
  854. for ring in polygon
  855. ]
  856. for polygon in coordinates
  857. ],
  858. }
  859. raise ValueError(f"Unsupported geometry type: {geometry_type}")
  860. def build_planar_group_offsets(territory_key: str, group_bounds: dict[str, tuple[float, float, float, float]]) -> dict[str, tuple[float, float]]:
  861. layout_rows = SPECIAL_PLANAR_LAYOUT_ROWS.get(territory_key)
  862. if not layout_rows or not group_bounds:
  863. return {}
  864. widths = [max_x - min_x for min_x, _min_y, max_x, _max_y in group_bounds.values()]
  865. heights = [max_y - min_y for _min_x, min_y, _max_x, max_y in group_bounds.values()]
  866. gap_x = (max(widths) if widths else 0) * 0.18
  867. gap_y = (max(heights) if heights else 0) * 0.28
  868. remaining_groups = [group_id for group_id in group_bounds if group_id not in {item for row in layout_rows for item in row}]
  869. rows = [row[:] for row in layout_rows]
  870. if remaining_groups:
  871. rows.append(remaining_groups)
  872. offsets: dict[str, tuple[float, float]] = {}
  873. cursor_y = 0.0
  874. for row in rows:
  875. row_groups = [group_id for group_id in row if group_id in group_bounds]
  876. if not row_groups:
  877. continue
  878. row_height = max(group_bounds[group_id][3] - group_bounds[group_id][1] for group_id in row_groups)
  879. cursor_x = 0.0
  880. for group_id in row_groups:
  881. min_x, min_y, max_x, max_y = group_bounds[group_id]
  882. width = max_x - min_x
  883. height = max_y - min_y
  884. target_min_x = cursor_x
  885. target_min_y = cursor_y + (row_height - height) / 2
  886. offsets[group_id] = (target_min_x - min_x, target_min_y - min_y)
  887. cursor_x += width + gap_x
  888. cursor_y += row_height + gap_y
  889. return offsets
  890. def apply_special_planar_layout(
  891. territory_key: str,
  892. raw_features: list[dict],
  893. projected_features: list[dict],
  894. raw_units: list[dict],
  895. projected_units: list[dict],
  896. ) -> tuple[list[dict], list[dict]]:
  897. if territory_key not in SPECIAL_PLANAR_LAYOUT_ROWS:
  898. return projected_features, projected_units
  899. projected_group_bounds: dict[str, list[float]] = {}
  900. feature_group_by_code: dict[str, str] = {}
  901. for raw_feature, projected_feature in zip(raw_features, projected_features):
  902. center_longitude, center_latitude = get_geometry_center(raw_feature["geometry"])
  903. group_id = get_special_layout_group_id(territory_key, center_longitude, center_latitude)
  904. if not group_id:
  905. continue
  906. feature_group_by_code[raw_feature["properties"]["c"]] = group_id
  907. min_x, min_y, max_x, max_y = get_geometry_bounds(projected_feature["geometry"])
  908. current_bounds = projected_group_bounds.get(group_id)
  909. if current_bounds is None:
  910. projected_group_bounds[group_id] = [min_x, min_y, max_x, max_y]
  911. else:
  912. current_bounds[0] = min(current_bounds[0], min_x)
  913. current_bounds[1] = min(current_bounds[1], min_y)
  914. current_bounds[2] = max(current_bounds[2], max_x)
  915. current_bounds[3] = max(current_bounds[3], max_y)
  916. offsets = build_planar_group_offsets(
  917. territory_key,
  918. {
  919. group_id: (bounds[0], bounds[1], bounds[2], bounds[3])
  920. for group_id, bounds in projected_group_bounds.items()
  921. },
  922. )
  923. if not offsets:
  924. return projected_features, projected_units
  925. translated_features = []
  926. for raw_feature, projected_feature in zip(raw_features, projected_features):
  927. group_id = feature_group_by_code.get(raw_feature["properties"]["c"])
  928. offset_x, offset_y = offsets.get(group_id, (0.0, 0.0))
  929. translated_features.append(
  930. {
  931. "type": "Feature",
  932. "properties": dict(projected_feature["properties"]),
  933. "geometry": translate_geometry(projected_feature["geometry"], offset_x, offset_y),
  934. }
  935. )
  936. translated_units = []
  937. for raw_unit, projected_unit in zip(raw_units, projected_units):
  938. group_id = get_special_layout_group_id(territory_key, raw_unit["x"], raw_unit["y"])
  939. offset_x, offset_y = offsets.get(group_id, (0.0, 0.0))
  940. translated_unit = dict(projected_unit)
  941. translated_unit["x"] = round(float(translated_unit["x"]) + offset_x, 6)
  942. translated_unit["y"] = round(float(translated_unit["y"]) + offset_y, 6)
  943. translated_units.append(translated_unit)
  944. return translated_features, translated_units
  945. def reduce_group_distances_without_overlap(
  946. territory_key: str,
  947. raw_features: list[dict],
  948. projected_features: list[dict],
  949. raw_units: list[dict],
  950. projected_units: list[dict],
  951. reduction_ratio: float,
  952. ) -> tuple[list[dict], list[dict]]:
  953. if reduction_ratio <= 0 or not projected_features:
  954. return projected_features, projected_units
  955. clamped_reduction_ratio = min(max(float(reduction_ratio), 0.0), 1.0)
  956. distance_factor = 1.0 - clamped_reduction_ratio
  957. feature_group_by_code: dict[str, str] = {}
  958. group_bounds: dict[str, list[float]] = {}
  959. group_centers: dict[str, tuple[float, float]] = {}
  960. pack_by_feature = SPECIAL_GROUP_PACK_BY_FEATURE.get(territory_key, False)
  961. for raw_feature, projected_feature in zip(raw_features, projected_features):
  962. center_longitude, center_latitude = get_geometry_center(raw_feature["geometry"])
  963. feature_code = str(raw_feature["properties"]["c"])
  964. group_id = feature_code if pack_by_feature else get_special_layout_group_id(
  965. territory_key, center_longitude, center_latitude
  966. )
  967. if not group_id:
  968. continue
  969. feature_group_by_code[feature_code] = group_id
  970. min_x, min_y, max_x, max_y = get_geometry_bounds(projected_feature["geometry"])
  971. current_bounds = group_bounds.get(group_id)
  972. if current_bounds is None:
  973. group_bounds[group_id] = [min_x, min_y, max_x, max_y]
  974. else:
  975. current_bounds[0] = min(current_bounds[0], min_x)
  976. current_bounds[1] = min(current_bounds[1], min_y)
  977. current_bounds[2] = max(current_bounds[2], max_x)
  978. current_bounds[3] = max(current_bounds[3], max_y)
  979. if not group_bounds:
  980. return projected_features, projected_units
  981. min_all_x = min(bounds[0] for bounds in group_bounds.values())
  982. min_all_y = min(bounds[1] for bounds in group_bounds.values())
  983. max_all_x = max(bounds[2] for bounds in group_bounds.values())
  984. max_all_y = max(bounds[3] for bounds in group_bounds.values())
  985. global_center_x = (min_all_x + max_all_x) / 2
  986. global_center_y = (min_all_y + max_all_y) / 2
  987. max_dimension = 0.0
  988. group_layout = {}
  989. for group_id, bounds in group_bounds.items():
  990. min_x, min_y, max_x, max_y = bounds
  991. center_x = (min_x + max_x) / 2
  992. center_y = (min_y + max_y) / 2
  993. width = max_x - min_x
  994. height = max_y - min_y
  995. max_dimension = max(max_dimension, width, height)
  996. target_x = global_center_x + (center_x - global_center_x) * distance_factor
  997. target_y = global_center_y + (center_y - global_center_y) * distance_factor
  998. group_centers[group_id] = (center_x, center_y)
  999. group_layout[group_id] = {
  1000. "center_x": target_x,
  1001. "center_y": target_y,
  1002. "target_x": target_x,
  1003. "target_y": target_y,
  1004. "width": width,
  1005. "height": height,
  1006. }
  1007. group_gap_factor = SPECIAL_GROUP_MIN_GAP_FACTOR.get(territory_key, 0.04)
  1008. effective_bounds_factor = SPECIAL_GROUP_EFFECTIVE_BOUNDS_FACTOR.get(territory_key, 1.0)
  1009. gap = max_dimension * group_gap_factor
  1010. group_ids = list(group_layout.keys())
  1011. for _ in range(180):
  1012. moved = False
  1013. for index in range(len(group_ids)):
  1014. left_group = group_layout[group_ids[index]]
  1015. for right_index in range(index + 1, len(group_ids)):
  1016. right_group = group_layout[group_ids[right_index]]
  1017. delta_x = right_group["center_x"] - left_group["center_x"]
  1018. delta_y = right_group["center_y"] - left_group["center_y"]
  1019. required_x = (
  1020. (left_group["width"] + right_group["width"]) * effective_bounds_factor / 2 + gap
  1021. )
  1022. required_y = (
  1023. (left_group["height"] + right_group["height"]) * effective_bounds_factor / 2 + gap
  1024. )
  1025. overlap_x = required_x - abs(delta_x)
  1026. overlap_y = required_y - abs(delta_y)
  1027. if overlap_x <= 0 or overlap_y <= 0:
  1028. continue
  1029. moved = True
  1030. if overlap_x < overlap_y:
  1031. direction_x = 1 if delta_x >= 0 else -1
  1032. shift = overlap_x / 2
  1033. left_group["center_x"] -= direction_x * shift
  1034. right_group["center_x"] += direction_x * shift
  1035. else:
  1036. direction_y = 1 if delta_y >= 0 else -1
  1037. shift = overlap_y / 2
  1038. left_group["center_y"] -= direction_y * shift
  1039. right_group["center_y"] += direction_y * shift
  1040. for group in group_layout.values():
  1041. group["center_x"] += (group["target_x"] - group["center_x"]) * 0.08
  1042. group["center_y"] += (group["target_y"] - group["center_y"]) * 0.08
  1043. if not moved:
  1044. break
  1045. for _ in range(120):
  1046. moved = False
  1047. for index in range(len(group_ids)):
  1048. left_group = group_layout[group_ids[index]]
  1049. for right_index in range(index + 1, len(group_ids)):
  1050. right_group = group_layout[group_ids[right_index]]
  1051. delta_x = right_group["center_x"] - left_group["center_x"]
  1052. delta_y = right_group["center_y"] - left_group["center_y"]
  1053. required_x = (
  1054. (left_group["width"] + right_group["width"]) * effective_bounds_factor / 2 + gap
  1055. )
  1056. required_y = (
  1057. (left_group["height"] + right_group["height"]) * effective_bounds_factor / 2 + gap
  1058. )
  1059. overlap_x = required_x - abs(delta_x)
  1060. overlap_y = required_y - abs(delta_y)
  1061. if overlap_x <= 0 or overlap_y <= 0:
  1062. continue
  1063. moved = True
  1064. if overlap_x < overlap_y:
  1065. direction_x = 1 if delta_x >= 0 else -1
  1066. shift = overlap_x / 2
  1067. left_group["center_x"] -= direction_x * shift
  1068. right_group["center_x"] += direction_x * shift
  1069. else:
  1070. direction_y = 1 if delta_y >= 0 else -1
  1071. shift = overlap_y / 2
  1072. left_group["center_y"] -= direction_y * shift
  1073. right_group["center_y"] += direction_y * shift
  1074. if not moved:
  1075. break
  1076. group_offsets = {}
  1077. for group_id, group in group_layout.items():
  1078. center_x, center_y = group_centers[group_id]
  1079. group_offsets[group_id] = (group["center_x"] - center_x, group["center_y"] - center_y)
  1080. translated_features = []
  1081. for raw_feature, projected_feature in zip(raw_features, projected_features):
  1082. group_id = feature_group_by_code.get(raw_feature["properties"]["c"])
  1083. offset_x, offset_y = group_offsets.get(group_id, (0.0, 0.0))
  1084. translated_features.append(
  1085. {
  1086. "type": "Feature",
  1087. "properties": dict(projected_feature["properties"]),
  1088. "geometry": translate_geometry(projected_feature["geometry"], offset_x, offset_y),
  1089. }
  1090. )
  1091. translated_units = []
  1092. for raw_unit, projected_unit in zip(raw_units, projected_units):
  1093. unit_commune_code = str(raw_unit.get("c", ""))
  1094. group_id = unit_commune_code if pack_by_feature and unit_commune_code else get_special_layout_group_id(
  1095. territory_key, raw_unit["x"], raw_unit["y"]
  1096. )
  1097. offset_x, offset_y = group_offsets.get(group_id, (0.0, 0.0))
  1098. translated_unit = dict(projected_unit)
  1099. translated_unit["x"] = round(float(translated_unit["x"]) + offset_x, 6)
  1100. translated_unit["y"] = round(float(translated_unit["y"]) + offset_y, 6)
  1101. translated_units.append(translated_unit)
  1102. return translated_features, translated_units
  1103. def optimize_special_territory_features(territory_key: str, features: list[dict]) -> list[dict]:
  1104. simplification = SPECIAL_TERRITORY_SIMPLIFICATION.get(territory_key)
  1105. source_centers = get_layout_source_centers(territory_key, features)
  1106. optimized_features = []
  1107. precision = SPECIAL_TERRITORY_QUANTIZATION.get(territory_key, 5)
  1108. for feature in features:
  1109. geometry = feature["geometry"]
  1110. commune_name = feature["properties"].get("n", "")
  1111. if simplification:
  1112. geometry = simplify_geometry(
  1113. geometry,
  1114. simplification["minDistance"],
  1115. simplification["minArea"],
  1116. )
  1117. geometry = trim_special_commune_polygons(territory_key, commune_name, geometry)
  1118. if source_centers:
  1119. geometry = transform_geometry_for_layout(territory_key, geometry, source_centers)
  1120. optimized_features.append(
  1121. {
  1122. "type": "Feature",
  1123. "properties": dict(feature["properties"]),
  1124. "geometry": quantize_geometry(geometry, precision),
  1125. }
  1126. )
  1127. return optimized_features
  1128. def optimize_special_territory_units(
  1129. territory_key: str,
  1130. units: list[dict],
  1131. source_centers: dict[str, tuple[float, float]],
  1132. ) -> list[dict]:
  1133. if not source_centers:
  1134. return [dict(unit) for unit in units]
  1135. optimized_units = []
  1136. for unit in units:
  1137. optimized_longitude, optimized_latitude = transform_point_for_layout(
  1138. territory_key,
  1139. unit["x"],
  1140. unit["y"],
  1141. source_centers,
  1142. )
  1143. optimized_unit = dict(unit)
  1144. optimized_unit["x"] = optimized_longitude
  1145. optimized_unit["y"] = optimized_latitude
  1146. optimized_units.append(optimized_unit)
  1147. return optimized_units
  1148. def fetch_department_commune_features(
  1149. department_code: str,
  1150. competence_by_commune_code: dict[str, dict] | None = None,
  1151. ) -> list[dict]:
  1152. query = urlencode(
  1153. {
  1154. "codeDepartement": department_code,
  1155. "fields": "nom,code,codeDepartement,codeRegion,centre,contour",
  1156. "format": "geojson",
  1157. "geometry": "contour",
  1158. }
  1159. )
  1160. url = f"{API_BASE_URL}/communes?{query}"
  1161. payload = fetch_json(url)
  1162. features = payload.get("features", [])
  1163. normalized_features = []
  1164. for feature in features:
  1165. properties = feature["properties"]
  1166. competence = (competence_by_commune_code or {}).get(properties["code"], {})
  1167. normalized_properties = {
  1168. "c": properties["code"],
  1169. "n": properties["nom"],
  1170. "d": properties["codeDepartement"],
  1171. "r": properties["codeRegion"],
  1172. "k": COMMUNE_ENTITY_TYPE,
  1173. }
  1174. if competence.get("i"):
  1175. normalized_properties["i"] = competence["i"]
  1176. if competence.get("u"):
  1177. normalized_properties["u"] = competence["u"]
  1178. if competence.get("m"):
  1179. normalized_properties["m"] = competence["m"]
  1180. if competence.get("v"):
  1181. normalized_properties["v"] = competence["v"]
  1182. normalized_features.append(
  1183. {
  1184. "type": "Feature",
  1185. "properties": normalized_properties,
  1186. "geometry": quantize_geometry(feature["geometry"]),
  1187. }
  1188. )
  1189. return normalized_features
  1190. def fetch_region_commune_features(
  1191. territory: dict,
  1192. competence_by_commune_code: dict[str, dict] | None = None,
  1193. ) -> list[dict]:
  1194. region_slug = territory["key"].removeprefix("region-")
  1195. url = (
  1196. f"{REGION_COMMUNES_GEOJSON_BASE_URL}/{region_slug}/"
  1197. f"communes-{region_slug}.geojson"
  1198. )
  1199. payload = fetch_json(url)
  1200. features = payload.get("features", [])
  1201. normalized_features = []
  1202. for feature in features:
  1203. properties = feature["properties"]
  1204. code = properties["code"]
  1205. competence = (competence_by_commune_code or {}).get(code, {})
  1206. normalized_properties = {
  1207. "c": code,
  1208. "n": properties["nom"],
  1209. "d": get_department_code_from_commune_code(code),
  1210. "r": territory["regionCode"],
  1211. "k": COMMUNE_ENTITY_TYPE,
  1212. }
  1213. if competence.get("i"):
  1214. normalized_properties["i"] = competence["i"]
  1215. if competence.get("u"):
  1216. normalized_properties["u"] = competence["u"]
  1217. if competence.get("m"):
  1218. normalized_properties["m"] = competence["m"]
  1219. if competence.get("v"):
  1220. normalized_properties["v"] = competence["v"]
  1221. normalized_features.append(
  1222. {
  1223. "type": "Feature",
  1224. "properties": normalized_properties,
  1225. "geometry": quantize_geometry(feature["geometry"]),
  1226. }
  1227. )
  1228. return normalized_features
  1229. def fetch_france_region_features(
  1230. region_metadata_by_code: dict[str, dict],
  1231. metro_region_codes: list[str],
  1232. region_key_by_code: dict[str, str],
  1233. ) -> list[dict]:
  1234. payload = fetch_json(REGIONS_GEOJSON_URL)
  1235. features_by_code = {}
  1236. for feature in payload.get("features", []):
  1237. code = feature["properties"]["code"]
  1238. if code not in region_metadata_by_code or code not in region_key_by_code:
  1239. continue
  1240. metadata = region_metadata_by_code[code]
  1241. features_by_code[code] = {
  1242. "type": "Feature",
  1243. "properties": {
  1244. "c": code,
  1245. "n": metadata["nom"],
  1246. "d": "",
  1247. "r": code,
  1248. "k": REGION_ENTITY_TYPE,
  1249. "x": region_key_by_code[code],
  1250. },
  1251. "geometry": quantize_geometry(feature["geometry"]),
  1252. }
  1253. return [features_by_code[code] for code in metro_region_codes if code in features_by_code]
  1254. def fetch_metropolitan_department_features(
  1255. department_metadata_by_code: dict[str, dict],
  1256. metro_department_codes: list[str],
  1257. ) -> list[dict]:
  1258. payload = fetch_json(DEPARTMENTS_GEOJSON_URL)
  1259. features_by_code = {}
  1260. for feature in payload.get("features", []):
  1261. code = feature["properties"]["code"]
  1262. if code not in department_metadata_by_code:
  1263. continue
  1264. metadata = department_metadata_by_code[code]
  1265. features_by_code[code] = {
  1266. "type": "Feature",
  1267. "properties": {
  1268. "c": code,
  1269. "n": metadata["nom"],
  1270. "d": code,
  1271. "r": metadata["codeRegion"],
  1272. "k": DEPARTMENT_ENTITY_TYPE,
  1273. "x": department_territory_key(code),
  1274. },
  1275. "geometry": quantize_geometry(feature["geometry"]),
  1276. }
  1277. return [features_by_code[code] for code in metro_department_codes if code in features_by_code]
  1278. def build_index_entries(
  1279. features: list[dict],
  1280. territory_key: str,
  1281. commune_postal_codes_by_code: dict[str, list[str]],
  1282. is_hidden: bool = False,
  1283. ) -> list[dict]:
  1284. entries = []
  1285. for feature in features:
  1286. properties = feature["properties"]
  1287. entry = {
  1288. "c": properties["c"],
  1289. "n": properties["n"],
  1290. "t": territory_key,
  1291. "k": properties["k"],
  1292. }
  1293. if properties["k"] == COMMUNE_ENTITY_TYPE:
  1294. entry["p"] = commune_postal_codes_by_code.get(properties["c"], [])
  1295. if is_hidden:
  1296. entry["h"] = 1
  1297. entries.append(entry)
  1298. return entries
  1299. def iter_outer_rings(geometry: dict) -> Iterable[list[list[float]]]:
  1300. geometry_type = geometry.get("type")
  1301. coordinates = geometry.get("coordinates", [])
  1302. if geometry_type == "Polygon":
  1303. if coordinates:
  1304. yield coordinates[0]
  1305. return
  1306. if geometry_type == "MultiPolygon":
  1307. for polygon in coordinates:
  1308. if polygon:
  1309. yield polygon[0]
  1310. def normalize_ring_points(ring: list[list[float]], precision: int = 6) -> list[tuple[float, float]]:
  1311. points = [
  1312. (round(float(point[0]), precision), round(float(point[1]), precision))
  1313. for point in ring
  1314. if isinstance(point, list) and len(point) >= 2
  1315. ]
  1316. if len(points) >= 2 and points[0] == points[-1]:
  1317. points = points[:-1]
  1318. return points
  1319. def build_edge_key(start: tuple[float, float], end: tuple[float, float]) -> tuple[tuple[float, float], tuple[float, float]]:
  1320. return (start, end) if start <= end else (end, start)
  1321. def build_global_edge_counts(features: list[dict], precision: int = 6) -> dict[tuple[tuple[float, float], tuple[float, float]], int]:
  1322. edge_counts: dict[tuple[tuple[float, float], tuple[float, float]], int] = {}
  1323. for feature in features:
  1324. for ring in iter_outer_rings(feature.get("geometry", {})):
  1325. points = normalize_ring_points(ring, precision)
  1326. point_count = len(points)
  1327. if point_count < 2:
  1328. continue
  1329. for index in range(point_count):
  1330. start = points[index]
  1331. end = points[(index + 1) % point_count]
  1332. if start == end:
  1333. continue
  1334. edge_key = build_edge_key(start, end)
  1335. edge_counts[edge_key] = edge_counts.get(edge_key, 0) + 1
  1336. return edge_counts
  1337. def build_maritime_lines_from_features(
  1338. features: list[dict],
  1339. edge_counts: dict[tuple[tuple[float, float], tuple[float, float]], int],
  1340. precision: int = 6,
  1341. ) -> list[list[list[float]]]:
  1342. maritime_lines: list[list[list[float]]] = []
  1343. for feature in features:
  1344. for ring in iter_outer_rings(feature.get("geometry", {})):
  1345. points = normalize_ring_points(ring, precision)
  1346. point_count = len(points)
  1347. if point_count < 2:
  1348. continue
  1349. edge_is_maritime = []
  1350. for index in range(point_count):
  1351. start = points[index]
  1352. end = points[(index + 1) % point_count]
  1353. edge_key = build_edge_key(start, end)
  1354. edge_is_maritime.append(edge_counts.get(edge_key, 0) == 1)
  1355. if not any(edge_is_maritime):
  1356. continue
  1357. start_indexes = [
  1358. index
  1359. for index in range(point_count)
  1360. if edge_is_maritime[index] and not edge_is_maritime[index - 1]
  1361. ]
  1362. if not start_indexes:
  1363. line_points = points + [points[0]]
  1364. maritime_lines.append([[point_x, point_y] for point_x, point_y in line_points])
  1365. continue
  1366. for start_index in start_indexes:
  1367. index = start_index
  1368. line_points = [points[start_index]]
  1369. while edge_is_maritime[index]:
  1370. next_index = (index + 1) % point_count
  1371. line_points.append(points[next_index])
  1372. index = next_index
  1373. if index == start_index:
  1374. break
  1375. if len(line_points) >= 2:
  1376. maritime_lines.append([[point_x, point_y] for point_x, point_y in line_points])
  1377. return maritime_lines
  1378. def iter_line_strings(geometry: dict) -> Iterable[list[list[float]]]:
  1379. geometry_type = geometry.get("type")
  1380. coordinates = geometry.get("coordinates", [])
  1381. if geometry_type == "LineString":
  1382. if coordinates:
  1383. yield coordinates
  1384. return
  1385. if geometry_type == "MultiLineString":
  1386. for line in coordinates:
  1387. if line:
  1388. yield line
  1389. def build_coastline_index() -> dict:
  1390. payload = fetch_json(COASTLINE_GEOJSON_URL)
  1391. segments = []
  1392. grid: dict[tuple[int, int], list[int]] = {}
  1393. for feature in payload.get("features", []):
  1394. for line in iter_line_strings(feature.get("geometry", {})):
  1395. for index in range(len(line) - 1):
  1396. start = line[index]
  1397. end = line[index + 1]
  1398. if len(start) < 2 or len(end) < 2:
  1399. continue
  1400. start_x, start_y = float(start[0]), float(start[1])
  1401. end_x, end_y = float(end[0]), float(end[1])
  1402. if start_x == end_x and start_y == end_y:
  1403. continue
  1404. min_x = min(start_x, end_x)
  1405. min_y = min(start_y, end_y)
  1406. max_x = max(start_x, end_x)
  1407. max_y = max(start_y, end_y)
  1408. segment_index = len(segments)
  1409. segments.append((start_x, start_y, end_x, end_y, min_x, min_y, max_x, max_y))
  1410. min_cell_x = math.floor(min_x / COASTLINE_GRID_SIZE)
  1411. max_cell_x = math.floor(max_x / COASTLINE_GRID_SIZE)
  1412. min_cell_y = math.floor(min_y / COASTLINE_GRID_SIZE)
  1413. max_cell_y = math.floor(max_y / COASTLINE_GRID_SIZE)
  1414. for cell_x in range(min_cell_x, max_cell_x + 1):
  1415. for cell_y in range(min_cell_y, max_cell_y + 1):
  1416. grid.setdefault((cell_x, cell_y), []).append(segment_index)
  1417. return {
  1418. "segments": segments,
  1419. "grid": grid,
  1420. }
  1421. def point_to_segment_distance_squared(
  1422. point_x: float,
  1423. point_y: float,
  1424. start_x: float,
  1425. start_y: float,
  1426. end_x: float,
  1427. end_y: float,
  1428. ) -> float:
  1429. delta_x = end_x - start_x
  1430. delta_y = end_y - start_y
  1431. length_squared = delta_x * delta_x + delta_y * delta_y
  1432. if length_squared <= 1e-18:
  1433. return (point_x - start_x) ** 2 + (point_y - start_y) ** 2
  1434. projection = ((point_x - start_x) * delta_x + (point_y - start_y) * delta_y) / length_squared
  1435. projection = max(0.0, min(1.0, projection))
  1436. nearest_x = start_x + projection * delta_x
  1437. nearest_y = start_y + projection * delta_y
  1438. return (point_x - nearest_x) ** 2 + (point_y - nearest_y) ** 2
  1439. def is_segment_near_coastline(
  1440. start: list[float],
  1441. end: list[float],
  1442. coastline_index: dict,
  1443. threshold_squared: float,
  1444. ) -> bool:
  1445. midpoint_x = (float(start[0]) + float(end[0])) / 2
  1446. midpoint_y = (float(start[1]) + float(end[1])) / 2
  1447. cell_x = math.floor(midpoint_x / COASTLINE_GRID_SIZE)
  1448. cell_y = math.floor(midpoint_y / COASTLINE_GRID_SIZE)
  1449. search_radius = 1
  1450. segments = coastline_index["segments"]
  1451. grid = coastline_index["grid"]
  1452. threshold = math.sqrt(threshold_squared)
  1453. candidate_indexes = set()
  1454. for offset_x in range(-search_radius, search_radius + 1):
  1455. for offset_y in range(-search_radius, search_radius + 1):
  1456. candidate_indexes.update(grid.get((cell_x + offset_x, cell_y + offset_y), []))
  1457. if not candidate_indexes:
  1458. return False
  1459. for segment_index in candidate_indexes:
  1460. start_x, start_y, end_x, end_y, min_x, min_y, max_x, max_y = segments[segment_index]
  1461. if (
  1462. midpoint_x < min_x - threshold
  1463. or midpoint_x > max_x + threshold
  1464. or midpoint_y < min_y - threshold
  1465. or midpoint_y > max_y + threshold
  1466. ):
  1467. continue
  1468. if (
  1469. point_to_segment_distance_squared(
  1470. midpoint_x,
  1471. midpoint_y,
  1472. start_x,
  1473. start_y,
  1474. end_x,
  1475. end_y,
  1476. )
  1477. <= threshold_squared
  1478. ):
  1479. return True
  1480. return False
  1481. def filter_maritime_lines_by_coastline(
  1482. maritime_lines: list[list[list[float]]],
  1483. coastline_index: dict,
  1484. threshold: float = COASTLINE_DISTANCE_THRESHOLD,
  1485. ) -> list[list[list[float]]]:
  1486. threshold_squared = threshold * threshold
  1487. filtered_lines: list[list[list[float]]] = []
  1488. for line in maritime_lines:
  1489. if len(line) < 2:
  1490. continue
  1491. segment_flags = [
  1492. is_segment_near_coastline(line[index], line[index + 1], coastline_index, threshold_squared)
  1493. for index in range(len(line) - 1)
  1494. ]
  1495. current_line: list[list[float]] = []
  1496. for index, is_maritime in enumerate(segment_flags):
  1497. if is_maritime:
  1498. if not current_line:
  1499. current_line = [line[index], line[index + 1]]
  1500. else:
  1501. current_line.append(line[index + 1])
  1502. elif current_line:
  1503. if len(current_line) >= 2:
  1504. filtered_lines.append(current_line)
  1505. current_line = []
  1506. if current_line and len(current_line) >= 2:
  1507. filtered_lines.append(current_line)
  1508. return filtered_lines
  1509. def write_json(path: Path, payload: dict | list) -> None:
  1510. path.parent.mkdir(parents=True, exist_ok=True)
  1511. with path.open("w", encoding="utf-8") as file:
  1512. json.dump(payload, file, ensure_ascii=False, separators=(",", ":"))
  1513. def clear_generated_files() -> None:
  1514. TERRITORY_DIR.mkdir(parents=True, exist_ok=True)
  1515. for path in TERRITORY_DIR.glob("*.json"):
  1516. path.unlink()
  1517. for path in [DATA_DIR / "communes-index.json", DATA_DIR / "territories.json"]:
  1518. if path.exists():
  1519. path.unlink()
  1520. def build_france_territory(metro_department_codes: list[str]) -> dict:
  1521. return {
  1522. "key": "france",
  1523. "label": "France",
  1524. "shortLabel": "France",
  1525. "entityType": REGION_ENTITY_TYPE,
  1526. "entityLabelSingular": "région de gendarmerie",
  1527. "entityLabelPlural": "régions de gendarmerie",
  1528. "departmentCodes": metro_department_codes,
  1529. "navVisible": True,
  1530. }
  1531. def build_region_territories(metro_regions: list[dict], department_codes_by_region: dict[str, list[str]]) -> list[dict]:
  1532. territories = []
  1533. for region in metro_regions:
  1534. territories.append(
  1535. {
  1536. "key": region_territory_key(region["nom"]),
  1537. "label": region["nom"],
  1538. "shortLabel": region["nom"],
  1539. "entityType": DEPARTMENT_ENTITY_TYPE,
  1540. "entityLabelSingular": "groupement",
  1541. "entityLabelPlural": "groupements",
  1542. "departmentCodes": department_codes_by_region.get(region["code"], []),
  1543. "regionCode": region["code"],
  1544. "navVisible": True,
  1545. }
  1546. )
  1547. return territories
  1548. def build_department_territories(metro_departments: list[dict], region_key_by_code: dict[str, str]) -> list[dict]:
  1549. territories = []
  1550. for department in metro_departments:
  1551. territories.append(
  1552. {
  1553. "key": department_territory_key(department["code"]),
  1554. "label": department["nom"],
  1555. "shortLabel": department["nom"],
  1556. "entityType": COMMUNE_ENTITY_TYPE,
  1557. "entityLabelSingular": "commune",
  1558. "entityLabelPlural": "communes",
  1559. "departmentCodes": [department["code"]],
  1560. "regionCode": department["codeRegion"],
  1561. "parentTerritoryKey": region_key_by_code[department["codeRegion"]],
  1562. "groupementView": True,
  1563. "navVisible": False,
  1564. }
  1565. )
  1566. return territories
  1567. def build_overseas_territories() -> list[dict]:
  1568. territories = []
  1569. for territory in OVERSEAS_TERRITORIES:
  1570. territories.append(
  1571. {
  1572. "key": slugify(territory["label"]),
  1573. "label": territory["label"],
  1574. "shortLabel": territory["shortLabel"],
  1575. "entityType": COMMUNE_ENTITY_TYPE,
  1576. "entityLabelSingular": "commune",
  1577. "entityLabelPlural": "communes",
  1578. "departmentCodes": [territory["code"]],
  1579. "groupementView": True,
  1580. "navVisible": True,
  1581. }
  1582. )
  1583. return territories
  1584. def build_data() -> None:
  1585. DATA_DIR.mkdir(parents=True, exist_ok=True)
  1586. TERRITORY_DIR.mkdir(parents=True, exist_ok=True)
  1587. clear_generated_files()
  1588. department_metadata = get_department_metadata()
  1589. region_metadata = get_region_metadata()
  1590. commune_postal_codes_by_code = get_commune_postal_codes_by_code()
  1591. competence_by_commune_code, service_registry, commune_codes_by_service_key = get_commune_territorial_competence_data()
  1592. public_units_by_service_key = get_gendarmerie_public_units_by_service_key(service_registry)
  1593. public_units_by_service_key.update(
  1594. get_police_public_units_by_service_key(service_registry, commune_codes_by_service_key)
  1595. )
  1596. department_metadata_by_code = {department["code"]: department for department in department_metadata}
  1597. region_metadata_by_code = {region["code"]: region for region in region_metadata}
  1598. metro_departments = [department for department in department_metadata if department.get("zone") == "metro"]
  1599. metro_department_codes = [department["code"] for department in metro_departments]
  1600. department_codes_by_region: dict[str, list[str]] = {}
  1601. for department in metro_departments:
  1602. department_codes_by_region.setdefault(department["codeRegion"], []).append(department["code"])
  1603. metro_regions = [
  1604. region
  1605. for region in region_metadata
  1606. if region["code"] in department_codes_by_region
  1607. ]
  1608. metro_region_codes = [region["code"] for region in metro_regions]
  1609. region_key_by_code = {region["code"]: region_territory_key(region["nom"]) for region in metro_regions}
  1610. france_territory = build_france_territory(metro_department_codes)
  1611. region_territories = build_region_territories(metro_regions, department_codes_by_region)
  1612. department_territories = build_department_territories(metro_departments, region_key_by_code)
  1613. overseas_territories = build_overseas_territories()
  1614. territory_definitions = [
  1615. france_territory,
  1616. *region_territories,
  1617. *overseas_territories,
  1618. *department_territories,
  1619. ]
  1620. france_region_features = fetch_france_region_features(
  1621. region_metadata_by_code,
  1622. metro_region_codes,
  1623. region_key_by_code,
  1624. )
  1625. print(f"France: {len(france_region_features)} régions")
  1626. metropolitan_department_features = fetch_metropolitan_department_features(
  1627. department_metadata_by_code,
  1628. metro_department_codes,
  1629. )
  1630. coastline_index = build_coastline_index()
  1631. region_department_features_by_key = {territory["key"]: [] for territory in region_territories}
  1632. for feature in metropolitan_department_features:
  1633. region_key = region_key_by_code.get(feature["properties"]["r"])
  1634. if region_key:
  1635. region_department_features_by_key[region_key].append(feature)
  1636. print(f"Départements métropolitains: {len(metropolitan_department_features)}")
  1637. department_commune_features_by_key = {territory["key"]: [] for territory in department_territories}
  1638. with ThreadPoolExecutor(max_workers=8) as executor:
  1639. future_by_key = {
  1640. executor.submit(fetch_region_commune_features, territory, competence_by_commune_code): territory["key"]
  1641. for territory in region_territories
  1642. }
  1643. for future in as_completed(future_by_key):
  1644. region_key = future_by_key[future]
  1645. features = future.result()
  1646. for feature in features:
  1647. department_key = department_territory_key(feature["properties"]["d"])
  1648. if department_key in department_commune_features_by_key:
  1649. department_commune_features_by_key[department_key].append(feature)
  1650. print(f"{region_key}: {len(features)} communes réparties par département")
  1651. overseas_commune_features_by_key: dict[str, list[dict]] = {}
  1652. overseas_masked_commune_features_by_key: dict[str, list[dict]] = {}
  1653. with ThreadPoolExecutor(max_workers=6) as executor:
  1654. future_by_key = {
  1655. executor.submit(
  1656. fetch_department_commune_features,
  1657. territory["departmentCodes"][0],
  1658. competence_by_commune_code,
  1659. ): territory["key"]
  1660. for territory in overseas_territories
  1661. }
  1662. for future in as_completed(future_by_key):
  1663. territory_key = future_by_key[future]
  1664. features = future.result()
  1665. if territory_key == "polynesie-francaise":
  1666. hidden_features = [
  1667. feature
  1668. for feature in features
  1669. if feature["properties"]["n"] not in POLYNESIE_RETAINED_COMMUNES
  1670. ]
  1671. features = [
  1672. feature
  1673. for feature in features
  1674. if feature["properties"]["n"] in POLYNESIE_RETAINED_COMMUNES
  1675. ]
  1676. overseas_masked_commune_features_by_key[territory_key] = hidden_features
  1677. overseas_commune_features_by_key[territory_key] = features
  1678. print(f"{territory_key}: {len(features)} communes")
  1679. territories_manifest = []
  1680. index_entries = []
  1681. for territory in territory_definitions:
  1682. masked_territory_features: list[dict] = []
  1683. if territory["key"] == "france":
  1684. territory_features = france_region_features
  1685. elif territory["entityType"] == DEPARTMENT_ENTITY_TYPE:
  1686. territory_features = region_department_features_by_key[territory["key"]]
  1687. elif territory.get("parentTerritoryKey"):
  1688. territory_features = department_commune_features_by_key[territory["key"]]
  1689. else:
  1690. territory_features = overseas_commune_features_by_key[territory["key"]]
  1691. masked_territory_features = overseas_masked_commune_features_by_key.get(territory["key"], [])
  1692. payload_features = territory_features
  1693. projected_unit_origin: tuple[float, float] | None = None
  1694. projected_unit_source: list[dict] = []
  1695. special_layout_source_centers: dict[str, tuple[float, float]] = {}
  1696. planar_compaction = PLANAR_TERRITORY_COMPACTION.get(territory["key"])
  1697. compact_factor = planar_compaction["factor"] if planar_compaction else 1.0
  1698. group_distance_reduction = SPECIAL_GROUP_DISTANCE_REDUCTION.get(territory["key"], 0.0)
  1699. if territory["key"] in SPECIAL_TERRITORY_LAYOUTS or territory["key"] in SPECIAL_TERRITORY_SIMPLIFICATION:
  1700. special_layout_source_centers = get_layout_source_centers(territory["key"], territory_features)
  1701. payload_features = optimize_special_territory_features(territory["key"], territory_features)
  1702. if territory["key"] in PREPROJECTED_TERRITORIES:
  1703. payload_features = [
  1704. {
  1705. "type": "Feature",
  1706. "properties": dict(feature["properties"]),
  1707. "geometry": project_geometry_to_planar_mercator(feature["geometry"]),
  1708. }
  1709. for feature in payload_features
  1710. ]
  1711. if compact_factor < 1:
  1712. payload_features, projected_unit_origin = compact_planar_features(payload_features, compact_factor)
  1713. payload = {
  1714. "type": "FeatureCollection",
  1715. "features": payload_features,
  1716. }
  1717. if territory["key"] in PREPROJECTED_TERRITORIES:
  1718. payload["coordinateSpace"] = (
  1719. "planar-layout" if territory["key"] in SPECIAL_PLANAR_LAYOUT_ROWS or compact_factor < 1 else "planar-mercator"
  1720. )
  1721. service_lookup = {}
  1722. unit_lookup = {}
  1723. for feature in territory_features:
  1724. service_keys = []
  1725. if feature["properties"].get("u"):
  1726. service_keys.append(feature["properties"]["u"])
  1727. service_keys.extend(feature["properties"].get("v", []))
  1728. for service_key in service_keys:
  1729. if service_key in service_registry:
  1730. service_lookup[service_key] = service_registry[service_key]
  1731. if service_key in public_units_by_service_key and territory.get("groupementView"):
  1732. unit_lookup[service_key] = public_units_by_service_key[service_key]
  1733. if service_lookup:
  1734. payload["services"] = service_lookup
  1735. if unit_lookup:
  1736. optimized_units = list(unit_lookup.values())
  1737. if special_layout_source_centers:
  1738. optimized_units = optimize_special_territory_units(
  1739. territory["key"],
  1740. optimized_units,
  1741. special_layout_source_centers,
  1742. )
  1743. if territory["key"] in PREPROJECTED_TERRITORIES:
  1744. projected_unit_source = optimized_units
  1745. optimized_units = project_units_to_planar_mercator(optimized_units)
  1746. if compact_factor < 1 and projected_unit_origin:
  1747. origin_x, origin_y = projected_unit_origin
  1748. optimized_units = compact_planar_units(optimized_units, origin_x, origin_y, compact_factor)
  1749. if territory["key"] in SPECIAL_PLANAR_LAYOUT_ROWS:
  1750. payload_features, optimized_units = apply_special_planar_layout(
  1751. territory["key"],
  1752. territory_features,
  1753. payload_features,
  1754. projected_unit_source or [],
  1755. optimized_units,
  1756. )
  1757. payload["features"] = payload_features
  1758. if group_distance_reduction > 0 and territory["key"] in PREPROJECTED_TERRITORIES:
  1759. payload_features, optimized_units = reduce_group_distances_without_overlap(
  1760. territory["key"],
  1761. territory_features,
  1762. payload_features,
  1763. projected_unit_source or [],
  1764. optimized_units,
  1765. group_distance_reduction,
  1766. )
  1767. payload["features"] = payload_features
  1768. payload["units"] = sorted(
  1769. optimized_units,
  1770. key=lambda item: (normalize_sort_text(item["n"]), item["n"].lower(), item["s"]),
  1771. )
  1772. elif territory["key"] in SPECIAL_PLANAR_LAYOUT_ROWS:
  1773. payload_features, _ = apply_special_planar_layout(
  1774. territory["key"],
  1775. territory_features,
  1776. payload_features,
  1777. [],
  1778. [],
  1779. )
  1780. payload["features"] = payload_features
  1781. if group_distance_reduction > 0 and territory["key"] in PREPROJECTED_TERRITORIES and not unit_lookup:
  1782. payload_features, _ = reduce_group_distances_without_overlap(
  1783. territory["key"],
  1784. territory_features,
  1785. payload["features"],
  1786. [],
  1787. [],
  1788. group_distance_reduction,
  1789. )
  1790. payload["features"] = payload_features
  1791. if masked_territory_features:
  1792. masked_communes = sorted(
  1793. [
  1794. {
  1795. "c": feature["properties"]["c"],
  1796. "n": feature["properties"]["n"],
  1797. "p": commune_postal_codes_by_code.get(feature["properties"]["c"], []),
  1798. }
  1799. for feature in masked_territory_features
  1800. ],
  1801. key=lambda item: (normalize_sort_text(item["n"]), item["n"].lower(), item["c"]),
  1802. )
  1803. masked_commune_name_by_code = {item["c"]: item["n"] for item in masked_communes}
  1804. masked_units_by_service_key: dict[str, dict] = {}
  1805. for feature in masked_territory_features:
  1806. properties = feature["properties"]
  1807. service_keys = []
  1808. if properties.get("u"):
  1809. service_keys.append(properties["u"])
  1810. service_keys.extend(properties.get("v", []))
  1811. for service_key in service_keys:
  1812. service = service_registry.get(service_key, {})
  1813. public_unit = public_units_by_service_key.get(service_key, {})
  1814. unit_label = public_unit.get("n") or service.get("l") or service_key
  1815. institution = public_unit.get("i") or service.get("i") or "UNK"
  1816. if service_key not in masked_units_by_service_key:
  1817. masked_units_by_service_key[service_key] = {
  1818. "s": service_key,
  1819. "i": institution,
  1820. "n": unit_label,
  1821. "c": set(),
  1822. }
  1823. masked_units_by_service_key[service_key]["c"].add(properties["c"])
  1824. masked_units = []
  1825. for unit in masked_units_by_service_key.values():
  1826. commune_codes = sorted(unit["c"])
  1827. commune_names = sorted(
  1828. {masked_commune_name_by_code.get(code, code) for code in commune_codes},
  1829. key=normalize_sort_text,
  1830. )
  1831. masked_units.append(
  1832. {
  1833. "s": unit["s"],
  1834. "i": unit["i"],
  1835. "n": unit["n"],
  1836. "c": commune_codes,
  1837. "m": commune_names,
  1838. }
  1839. )
  1840. masked_units.sort(key=lambda item: (normalize_sort_text(item["n"]), item["n"].lower(), item["s"]))
  1841. payload["masked"] = {
  1842. "communes": masked_communes,
  1843. "units": masked_units,
  1844. }
  1845. territory_edge_counts = build_global_edge_counts(payload["features"])
  1846. maritime_lines = build_maritime_lines_from_features(payload["features"], territory_edge_counts)
  1847. if maritime_lines:
  1848. filtered_maritime_lines = filter_maritime_lines_by_coastline(maritime_lines, coastline_index)
  1849. if filtered_maritime_lines:
  1850. maritime_lines = filtered_maritime_lines
  1851. elif not territory.get("parentTerritoryKey") and territory["entityType"] == COMMUNE_ENTITY_TYPE:
  1852. # Overseas islands can be too small for coarse coastline matching; keep raw maritime edges.
  1853. pass
  1854. else:
  1855. maritime_lines = []
  1856. if maritime_lines:
  1857. payload["maritime"] = maritime_lines
  1858. write_json(TERRITORY_DIR / f"{territory['key']}.json", payload)
  1859. territories_manifest.append(
  1860. {
  1861. "key": territory["key"],
  1862. "label": territory["label"],
  1863. "shortLabel": territory["shortLabel"],
  1864. "departmentCodes": territory["departmentCodes"],
  1865. "entityType": territory["entityType"],
  1866. "entityLabelSingular": territory["entityLabelSingular"],
  1867. "entityLabelPlural": territory["entityLabelPlural"],
  1868. "entityCount": len(territory_features),
  1869. "file": f"./data/territories/{territory['key']}.json",
  1870. "navVisible": territory.get("navVisible", True),
  1871. "parentTerritoryKey": territory.get("parentTerritoryKey"),
  1872. "groupementView": territory.get("groupementView", False),
  1873. "regionCode": territory.get("regionCode"),
  1874. }
  1875. )
  1876. index_entries.extend(
  1877. build_index_entries(
  1878. territory_features,
  1879. territory["key"],
  1880. commune_postal_codes_by_code,
  1881. )
  1882. )
  1883. if masked_territory_features:
  1884. index_entries.extend(
  1885. build_index_entries(
  1886. masked_territory_features,
  1887. territory["key"],
  1888. commune_postal_codes_by_code,
  1889. is_hidden=True,
  1890. )
  1891. )
  1892. print(
  1893. f"{territory['label']}: {len(territory_features)} {territory['entityLabelPlural']}"
  1894. )
  1895. index_entries.sort(key=lambda item: (normalize_sort_text(item["n"]), item["n"].lower(), item["c"], item["t"]))
  1896. write_json(DATA_DIR / "communes-index.json", index_entries)
  1897. write_json(DATA_DIR / "territories.json", territories_manifest)
  1898. print(f"Index written: {len(index_entries)} entrées")
  1899. if __name__ == "__main__":
  1900. build_data()