#!/usr/bin/env python3 """ Pass 8: Extract structured JSON data from ideal-investisseur SCPI pages. The pages contain ', html, re.S) for block in blocks: try: data = json.loads(block) if 'latest' in data: return data except: pass return None def json_to_sheet(data): """Map JSON fields to sheet columns""" d = {} latest = data.get('latest', {}) scpi_data = data.get('scpi', {}) # K: Endettement v = latest.get('taux_endettement') if v and v != '0' and v != '0.00': d['K'] = str(v) + '%' # S: TOF v = latest.get('tof') if v and v != '0' and v != '0.00': d['S'] = str(v) + '%' # V: Prix souscription v = latest.get('prix_souscription') if v: d['V'] = str(v) + ' €' # W: Valeur reconstitution v = latest.get('valeur_reconstitution') if v and str(v) != '0' and str(v) != '0.00': d['W'] = str(v) + ' €' # T: TD (taux distribution = performance globale) v = latest.get('taux_distribution') if v and str(v) != '0' and str(v) != '0.00': d['T'] = str(v) + '%' # U: TRI 10 ans v = latest.get('tri_10_ans') if v and str(v) != '0' and str(v) != '0.00': d['U'] = str(v) + '%' # G: Label ISR v = latest.get('label_isr') if v is not None: d['G'] = 'Oui' if str(v).lower() not in ('null', '', 'non', '0') else 'Non' # J: Capitalisation v = latest.get('capitalisation') if v: d['J'] = str(v) # M: Parts retrait v = latest.get('parts_en_attente_retrait') if v and str(v) != '0' and str(v) != 'null': d['M'] = str(v) # Dividende annuel (for context) v = latest.get('dividende_annuel') return d # Read sheet rows = read_sheet() needs = {} for i in range(1, len(rows)): row = rows[i] r = i + 1 scpi = row[1].strip() if len(row) > 1 else '' if not scpi: continue empty = {} for j in range(3, 25): val = row[j].strip() if j < len(row) else '' if val in ('', 'N/A', 'Non trouvé', '-'): col = chr(65+j) empty[col] = j if empty: needs[r] = (scpi, empty) print(f"Rows needing fixes: {len(needs)}") print(f"URLs available: {len(URL_MAP)}") fetcher = Fetcher(auto_match=False) total_cells = 0 total_rows = 0 no_url = [] no_json = [] for r, (scpi, empty) in sorted(needs.items()): if scpi not in URL_MAP: no_url.append(scpi) continue url = URL_MAP[scpi] try: page = fetcher.get(url, headers={'User-Agent': 'Mozilla/5.0 (compatible; Googlebot/2.1)'}) if page.status != 200 or not page.body or len(page.body) < 500: no_json.append(f"{scpi} (HTTP {page.status})") continue html = page.body.decode('utf-8', errors='ignore') data = extract_json_data(html) if not data or 'latest' not in data: no_json.append(f"{scpi} (no JSON)") continue except Exception as e: no_json.append(f"{scpi} (error: {e})") continue mapped = json_to_sheet(data) updates = [(col, mapped[col]) for col in empty if col in mapped] if not updates: no_json.append(f"{scpi} (no matching fields)") continue print(f"Row {r:3d} ({scpi:35s}): ", end="", flush=True) row_cells = 0 for col, value in updates: if gws_update(f"{col}{r}", value): print(f"{col}✅ ", end="", flush=True) row_cells += 1 time.sleep(0.15) else: print(f"{col}❌ ", end="", flush=True) print(f"({row_cells})") total_rows += 1 total_cells += row_cells time.sleep(0.3) print(f"\n\n🏁 TOTAL: {total_rows} rows, {total_cells} cells filled via JSON extraction") print(f"\nNo URL ({len(no_url)}): {', '.join(no_url[:10])}") print(f"No JSON ({len(no_json)}): {', '.join(no_json[:15])}")