import xml.etree.ElementTree as ET
import re
import json

file_path = "/home/user/uploads/thediscinitiatives.WordPress.2025-08-09.xml"
namespaces = {'wp': 'http://wordpress.org/export/1.2/', 'content': 'http://purl.org/rss/1.0/modules/content/'}

tree = ET.parse(file_path)
root = tree.getroot()

extracted_data = {}

def clean_html(raw_html):
    if not raw_html: return ""
    # Remove all shortcodes like [elementor-template id="123"]
    cleanr = re.compile(r'\[.*?\]')
    cleantext = re.sub(cleanr, '', raw_html)
    # Remove all HTML tags
    cleanr2 = re.compile(r'<.*?>')
    cleantext = re.sub(cleanr2, ' ', cleantext)
    # Collapse multiple spaces and newlines
    cleantext = re.sub(r'\s+', ' ', cleantext)
    return cleantext.strip()

for item in root.findall('.//item'):
    post_type_elem = item.find('wp:post_type', namespaces)
    status_elem = item.find('wp:status', namespaces)
    
    pt = post_type_elem.text if post_type_elem is not None else 'unknown'
    st = status_elem.text if status_elem is not None else 'unknown'
    
    if st == 'publish' and pt in ['page', 'team']:
        title = item.find('title').text
        content = item.find('content:encoded', namespaces)
        content_text = content.text if content is not None else ""
        
        # Check if the title is meaningful
        if not title:
            continue
            
        cleaned_content = clean_html(content_text)
        
        if pt not in extracted_data:
            extracted_data[pt] = []
            
        extracted_data[pt].append({
            'title': title,
            'content': cleaned_content
        })

with open('extracted_content.json', 'w') as f:
    json.dump(extracted_data, f, indent=2)

