import os
import glob
import requests
import pandas as pd
from bs4 import BeautifulSoup
import urllib.parse
import re

# Define directories
working_dir = "working_dir"          # Input CSV files directory
output_dir = "output_dir"            # Directory for the single output file
image_dir = "downloaded_images"      # Directory for downloaded images

# Ensure directories exist
os.makedirs(working_dir, exist_ok=True)
os.makedirs(output_dir, exist_ok=True)
os.makedirs(image_dir, exist_ok=True)

# Output file path
output_file = os.path.join(output_dir, "full-stock-list-update.csv")

# Fake browser headers
headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/134.0.0.0 Safari/537.36"
}

# Function to download an image with a modified filename
def download_image(image_url):
    original_filename = os.path.basename(urllib.parse.urlparse(image_url).path)

    # Replace "HED" with "TOP" and change ".jpg.webp" to ".jpg"
    new_filename = re.sub(r'^HED', 'TOP', original_filename, flags=re.IGNORECASE)
    if new_filename.lower().endswith(".jpg.webp"):
        new_filename = new_filename[:-8] + ".jpg"  # Remove ".webp" and keep ".jpg"

    # Remove any double dots
    new_filename = new_filename.replace("..", ".")

    filepath = os.path.join(image_dir, new_filename)

    # Check if the file already exists
    if os.path.exists(filepath):
        print(f"Image already exists, skipping: {new_filename}")
        return new_filename

    response = requests.get(image_url, headers=headers, stream=True)
    if response.status_code == 200:
        with open(filepath, 'wb') as file:
            for chunk in response.iter_content(1024):
                file.write(chunk)
        print(f"Downloaded: {new_filename}")
        return new_filename
    else:
        print(f"Failed to download: {image_url}")
        return None

# Process all CSV files in "working_dir"
csv_files = glob.glob(os.path.join(working_dir, "*.csv"))
#csv_files = glob.glob(os.path.join(working_dir, "full-stock-list25.csv"))

if not csv_files:
    print("No CSV files found in 'working_dir'.")

all_data = []  # List to store all rows before writing to the output file

for csv_file in csv_files:
    print(f"Processing file: {csv_file}")

    try:
        df = pd.read_csv(csv_file)

        if 'Link' not in df.columns:
            print(f"Column 'Link' not found in {csv_file}, skipping...")
            continue

        new_links = []
        for index, row in df.iterrows():
            url = row['Link']
            print(f"Processing URL: {url}")

            response = requests.get(url, headers=headers)
            if response.status_code == 200:
                soup = BeautifulSoup(response.text, 'html.parser')
                images = soup.find_all("img")

                image_downloaded = False
                for img in images:
                    img_url = img.get("src")
                    if img_url:
                        full_url = urllib.parse.urljoin(url, img_url)
                        filename = os.path.basename(urllib.parse.urlparse(full_url).path)

                        if "large" in full_url or "original" in full_url or "full" in full_url:
                            if re.match(r"^HED.*\.jpg\.webp$", filename, re.IGNORECASE):
                                new_filename = download_image(full_url)

                                if new_filename:
                                    new_link = f"https://toppersdrinks.co.uk/images/wines/{new_filename}"
                                    new_links.append(new_link)
                                    image_downloaded = True
                                    break

                if not image_downloaded:
                    new_links.append("")

            else:
                print(f"Failed to access page: {url}, Status Code: {response.status_code}")
                new_links.append("")

        if len(new_links) != len(df):
            raise ValueError("The number of new links does not match the number of rows in the original CSV.")

        # Add new column
        df['New Link'] = new_links

        # Modify "code" column to replace "HED" with "TOP"
        if 'code' in df.columns:
            df['code'] = df['code'].astype(str).apply(lambda x: re.sub(r'^HED', 'TOP', x, flags=re.IGNORECASE) if x.startswith('HED') else x)

        # Append data to list before writing
        all_data.append(df)

        # Rename processed file to ".prs"
        new_prs_filename = csv_file.replace(".csv", ".prs")
        os.rename(csv_file, new_prs_filename)
        print(f"Renamed processed file: {new_prs_filename}")

    except Exception as e:
        print(f"Error processing {csv_file}: {e}")

# Save final output to a single CSV file
if all_data:
    final_df = pd.concat(all_data, ignore_index=True)

    # Append to existing file or create a new one
    if os.path.exists(output_file):
        final_df.to_csv(output_file, mode='a', index=False, header=False)  # Append without header
        print(f"Appended data to: {output_file}")
    else:
        final_df.to_csv(output_file, index=False)  # Create new file with headers
        print(f"Created new output file: {output_file}")