import os
import shutil
from datetime import datetime

# Source and Destination paths
src_dir = r"e:\Wiki\knowledge"
dst_base = r"e:\Wiki\2nd\00_Raw"
today_folder = datetime.now().strftime("%Y-%m-%d")
dst_dir = os.path.join(dst_base, today_folder)

def ingest():
    if not os.path.exists(dst_dir):
        os.makedirs(dst_dir)
        
    # Get list of existing files in ANY subfolder of 00_Raw to avoid duplicates
    existing_files = set()
    for root, dirs, files in os.walk(dst_base):
        for f in files:
            existing_files.add(f)
            
    # Count how many files are ingested
    ingested_count = 0
    
    # Iterate through src_dir
    for root, dirs, files in os.walk(src_dir):
        for f in files:
            if f.endswith(".md") and f not in existing_files:
                src_path = os.path.join(root, f)
                dst_path = os.path.join(dst_dir, f)
                
                print(f"Ingesting: {f}")
                shutil.copy2(src_path, dst_path)
                ingested_count += 1
                
    print(f"Total Ingested: {ingested_count}")

if __name__ == "__main__":
    ingest()