Link to Google Drive -Folder Sync Using Google Colab

Markdown Detected hookstar1993 4 Views Size: 20.35 KB Posted on: Sep 14, 26 @ 5:00 AM

1--------------------------------------------------------------------

!pip -q install --upgrade google-api-python-client google-auth-httplib2 google-auth-oauthlib

2---------------------------------------------------------------------

from google.colab import auth auth.authenticate_user()

import google.auth from googleapiclient.discovery import build from googleapiclient.errors import HttpError import time

creds, _ = google.auth.default( scopes=["https://www.googleapis.com/auth/drive"] )

drive = build( "drive", "v3", credentials=creds )

print("✅ Google Drive connected")

3-------------------------------------------------------------

SOURCE_URLS = [ "PAST YOUR LINKS", "PAST YOUR LINKS", "PAST YOUR LINKS" ]

print("✅ 3 source folders loaded")

4------------------------------------------------------------------

DESTINATION_FOLDER_NAME = input( "Destination folder ka naam: " ).strip()

if not DESTINATION_FOLDER_NAME: raise ValueError("Destination folder name empty hai.")

print("Destination:", DESTINATION_FOLDER_NAME)

6-----------------------------------------------------------------------

============================================================

GOOGLE DRIVE SERVER-SIDE 3 FOLDER SYNC

============================================================

IMPORTANT:

Files are copied directly inside Google Drive.

They are NOT downloaded to Colab and re-uploaded.

Features:

- Recursive folders

- Large files

- Google Docs/Sheets/Slides preserved

- Existing files replaced

- Identical files skipped

- Human-readable file sizes

- Verification after copy

- Resume by simply running again after interruption

============================================================

import re import time from googleapiclient.errors import HttpError

------------------------------------------------------------

SETTINGS

------------------------------------------------------------

MAX_RETRIES = 8

FOLDER_MIME = "application/vnd.google-apps.folder"

------------------------------------------------------------

FOLDER ID FROM URL

------------------------------------------------------------

def get_folder_id(url):

match = re.search(
    r"/folders/([a-zA-Z0-9_-]+)",
    url
)

if not match:
    raise ValueError(
        f"Invalid Google Drive folder URL:\n{url}"
    )

return match.group(1)

SOURCE_FOLDER_IDS = [ get_folder_id(url) for url in SOURCE_URLS ]

------------------------------------------------------------

HUMAN READABLE SIZE

------------------------------------------------------------

def format_size(size):

if size is None:
    return "N/A"

size = int(size)

units = [
    "B",
    "KB",
    "MB",
    "GB",
    "TB",
    "PB"
]

value = float(size)

for unit in units:

    if value < 1024 or unit == units[-1]:

        if unit == "B":
            return f"{int(value):,} {unit}"

        return f"{value:,.2f} {unit}"

    value /= 1024

------------------------------------------------------------

GET FILE/FOLDER

------------------------------------------------------------

def get_item(file_id):

return drive.files().get(
    fileId=file_id,
    fields=(
        "id,"
        "name,"
        "mimeType,"
        "size,"
        "md5Checksum,"
        "modifiedTime,"
        "parents,"
        "trashed"
    ),
    supportsAllDrives=True
).execute()

------------------------------------------------------------

LIST ALL CHILDREN

------------------------------------------------------------

def list_children(parent_id):

all_items = []

page_token = None

while True:

    response = drive.files().list(

        q=(
            f"'{parent_id}' in parents "
            f"and trashed = false"
        ),

        spaces="drive",

        fields=(
            "nextPageToken,"
            "files("
            "id,"
            "name,"
            "mimeType,"
            "size,"
            "md5Checksum,"
            "modifiedTime,"
            "parents"
            ")"
        ),

        pageSize=1000,

        pageToken=page_token,

        supportsAllDrives=True,

        includeItemsFromAllDrives=True

    ).execute()

    all_items.extend(
        response.get("files", [])
    )

    page_token = response.get(
        "nextPageToken"
    )

    if not page_token:
        break

return all_items

------------------------------------------------------------

FIND ROOT DESTINATION FOLDER

------------------------------------------------------------

def find_root_folder(name):

safe_name = name.replace("'", "\\'")

response = drive.files().list(

    q=(
        f"name = '{safe_name}' "
        f"and mimeType = '{FOLDER_MIME}' "
        f"and 'root' in parents "
        f"and trashed = false"
    ),

    spaces="drive",

    fields="files(id,name,mimeType)",

    pageSize=100,

    supportsAllDrives=True,

    includeItemsFromAllDrives=True

).execute()

folders = response.get(
    "files",
    []
)

if folders:
    return folders[0]

return None

------------------------------------------------------------

CREATE FOLDER

------------------------------------------------------------

def create_folder(name, parent_id):

metadata = {

    "name": name,

    "mimeType": FOLDER_MIME,

    "parents": [
        parent_id
    ]

}

return drive.files().create(

    body=metadata,

    fields="id,name,mimeType,parents",

    supportsAllDrives=True

).execute()

------------------------------------------------------------

GET OR CREATE FOLDER

------------------------------------------------------------

def get_or_create_folder( name, parent_id, folder_cache ):

cache_key = (
    parent_id,
    name
)

if cache_key in folder_cache:

    return folder_cache[
        cache_key
    ]

children = list_children(
    parent_id
)

for item in children:

    if (
        item["mimeType"] == FOLDER_MIME
        and item["name"] == name
    ):

        folder_cache[
            cache_key
        ] = item

        return item

print(
    f"📁 Creating folder: {name}"
)

new_folder = create_folder(
    name,
    parent_id
)

folder_cache[
    cache_key
] = new_folder

return new_folder

------------------------------------------------------------

FILE MATCHING

------------------------------------------------------------

def find_existing_file( name, parent_id, children=None ):

if children is None:
    children = list_children(
        parent_id
    )

for item in children:

    if (
        item["mimeType"] != FOLDER_MIME
        and item["name"] == name
    ):

        return item

return None

------------------------------------------------------------

CHECK WHETHER FILES ARE IDENTICAL

------------------------------------------------------------

def files_are_identical( source, destination ):

# --------------------------------------------------------
# NORMAL FILES
# --------------------------------------------------------

source_md5 = source.get(
    "md5Checksum"
)

destination_md5 = destination.get(
    "md5Checksum"
)

if source_md5 and destination_md5:

    return (
        source_md5 ==
        destination_md5
    )

# --------------------------------------------------------
# GOOGLE WORKSPACE FILES
#
# Docs / Sheets / Slides do not normally have MD5.
# Compare modification time and name as fallback.
# --------------------------------------------------------

source_size = source.get(
    "size"
)

destination_size = destination.get(
    "size"
)

if (
    source_size is not None
    and destination_size is not None
):

    return (
        int(source_size)
        ==
        int(destination_size)
    )

return False

------------------------------------------------------------

SERVER SIDE COPY

------------------------------------------------------------

def copy_file( source_file, destination_parent_id, existing_file=None ):

source_id = source_file["id"]

source_name = source_file["name"]

# --------------------------------------------------------
# RETRY
# --------------------------------------------------------

for attempt in range(
    1,
    MAX_RETRIES + 1
):

    try:

        print(
            f"\n📄 {source_name}"
        )

        source_size = source_file.get(
            "size"
        )

        print(
            "   Source size:",
            format_size(source_size)
        )

        # ------------------------------------------------
        # COPY TO DESTINATION
        # ------------------------------------------------

        copied = drive.files().copy(

            fileId=source_id,

            body={
                "name": source_name,
                "parents": [
                    destination_parent_id
                ]
            },

            fields=(
                "id,"
                "name,"
                "mimeType,"
                "size,"
                "md5Checksum,"
                "modifiedTime,"
                "parents"
            ),

            supportsAllDrives=True

        ).execute()

        copied_id = copied["id"]

        # ------------------------------------------------
        # VERIFY COPIED FILE
        # ------------------------------------------------

        verified = get_item(
            copied_id
        )

        destination_size = verified.get(
            "size"
        )

        print(
            "   Copied size :",
            format_size(destination_size)
        )

        # ------------------------------------------------
        # VERIFY NORMAL FILE
        # ------------------------------------------------

        source_md5 = source_file.get(
            "md5Checksum"
        )

        copied_md5 = verified.get(
            "md5Checksum"
        )

        if source_md5 and copied_md5:

            if source_md5 != copied_md5:

                # Copy failed verification.
                # Delete bad copy.

                drive.files().update(
                    fileId=copied_id,
                    body={
                        "trashed": True
                    },
                    supportsAllDrives=True
                ).execute()

                raise Exception(
                    "MD5 verification failed"
                )

            print(
                "   MD5         : ✅ MATCH"
            )

        # ------------------------------------------------
        # VERIFY SIZE
        # ------------------------------------------------

        if (
            source_size is not None
            and destination_size is not None
        ):

            if (
                int(source_size)
                !=
                int(destination_size)
            ):

                drive.files().update(
                    fileId=copied_id,
                    body={
                        "trashed": True
                    },
                    supportsAllDrives=True
                ).execute()

                raise Exception(
                    "File size verification failed"
                )

            print(
                "   Size check  : ✅ MATCH"
            )

        # ------------------------------------------------
        # NOW REMOVE OLD VERSION
        # ------------------------------------------------

        if existing_file:

            print(
                "   Old file    : 🗑️ Replacing"
            )

            drive.files().update(

                fileId=existing_file["id"],

                body={
                    "trashed": True
                },

                supportsAllDrives=True

            ).execute()

            print(
                "   Result      : 🔄 REPLACED"
            )

        else:

            print(
                "   Result      : ✅ COPIED"
            )

        return copied

    except Exception as e:

        print(
            f"   ⚠️ Attempt "
            f"{attempt}/{MAX_RETRIES}: {e}"
        )

        if attempt < MAX_RETRIES:

            time.sleep(
                min(
                    2 ** attempt,
                    30
                )
            )

        else:

            raise

------------------------------------------------------------

STATISTICS

------------------------------------------------------------

stats = {

"folders_created": 0,

"files_copied": 0,

"files_replaced": 0,

"files_skipped": 0,

"errors": 0

}

------------------------------------------------------------

RECURSIVE SYNC

------------------------------------------------------------

def sync_folder( source_folder_id, destination_folder_id, current_path="" ):

children = list_children(
    source_folder_id
)

# --------------------------------------------------------
# PROCESS FOLDERS FIRST
# --------------------------------------------------------

for item in children:

    if item["mimeType"] != FOLDER_MIME:
        continue

    folder_name = item["name"]

    destination_folder = (
        get_or_create_folder(
            folder_name,
            destination_folder_id,
            folder_cache
        )
    )

    stats["folders_created"] += 1

    new_path = (
        f"{current_path}/{folder_name}"
        if current_path
        else folder_name
    )

    print(
        f"\n📁 Folder: {new_path}"
    )

    sync_folder(

        item["id"],

        destination_folder["id"],

        new_path

    )

# --------------------------------------------------------
# PROCESS FILES
# --------------------------------------------------------

for item in children:

    if item["mimeType"] == FOLDER_MIME:
        continue

    file_name = item["name"]

    existing = find_existing_file(

        file_name,

        destination_folder_id,

        children=destination_children_cache.get(
            destination_folder_id
        )

    )

    # ----------------------------------------------------
    # CACHE DESTINATION
    # ----------------------------------------------------

    if (
        destination_folder_id
        not in
        destination_children_cache
    ):

        destination_children_cache[
            destination_folder_id
        ] = list_children(
            destination_folder_id
        )

        existing = find_existing_file(

            file_name,

            destination_folder_id,

            destination_children_cache[
                destination_folder_id
            ]

        )

    # ----------------------------------------------------
    # ALREADY EXISTS
    # ----------------------------------------------------

    if existing:

        if files_are_identical(
            item,
            existing
        ):

            print(
                f"\n⏭️ SKIP: {file_name}"
            )

            print(
                "   Size:",
                format_size(
                    item.get("size")
                )
            )

            stats[
                "files_skipped"
            ] += 1

            continue

    # ----------------------------------------------------
    # COPY / REPLACE
    # ----------------------------------------------------

    try:

        result = copy_file(

            item,

            destination_folder_id,

            existing_file=existing

        )

        if existing:

            stats[
                "files_replaced"
            ] += 1

        else:

            stats[
                "files_copied"
            ] += 1

        # Update destination cache
        destination_children_cache[
            destination_folder_id
        ] = list_children(
            destination_folder_id
        )

    except Exception as e:

        stats[
            "errors"
        ] += 1

        print(
            f"\n❌ FAILED: {file_name}"
        )

        print(
            "   Error:",
            e
        )

------------------------------------------------------------

CREATE / FIND DESTINATION

------------------------------------------------------------

folder_cache = {}

destination_children_cache = {}

destination_root = find_root_folder( DESTINATION_FOLDER_NAME )

if destination_root:

print(
    "\n✅ Destination already exists:"
)

print(
    destination_root["name"]
)

else:

print(
    "\n📁 Creating destination:"
)

destination_root = create_folder(

    DESTINATION_FOLDER_NAME,

    "root"

)

print(
    "✅ Destination created"
)

DESTINATION_ROOT_ID = ( destination_root["id"] )

------------------------------------------------------------

RUN ALL THREE SOURCE FOLDERS

------------------------------------------------------------

print("\n") print("=" 75) print("🚀 STARTING 3-FOLDER GOOGLE DRIVE SYNC") print("=" 75)

for index, source_id in enumerate( SOURCE_FOLDER_IDS, start=1 ):

print("\n")
print("=" * 75)
print(
    f"📦 SOURCE {index} / 3"
)
print("=" * 75)

try:

    source_info = get_item(
        source_id
    )

    print(
        "Source folder:",
        source_info["name"]
    )

    sync_folder(

        source_id,

        DESTINATION_ROOT_ID,

        source_info["name"]

    )

except Exception as e:

    stats["errors"] += 1

    print(
        f"❌ SOURCE {index} FAILED:"
    )

    print(e)

------------------------------------------------------------

FINAL REPORT

------------------------------------------------------------

print("\n") print("=" 75) print("🎉 SYNC FINISHED") print("=" 75)

print( "📁 Folders processed :", stats["folders_created"] )

print( "📤 New files copied :", stats["files_copied"] )

print( "🔄 Files replaced :", stats["files_replaced"] )

print( "⏭️ Files skipped :", stats["files_skipped"] )

print( "❌ Errors :", stats["errors"] )

print("=" * 75)

Raw Paste

Comments 0
Login to post a comment.
  • No comments yet. Be the first.
Login to post a comment. Login or Register
We use cookies. To comply with GDPR in the EU and the UK we have to show you these.

We use cookies and similar technologies to keep this website functional (including spam protection via Google reCAPTCHA or Cloudflare Turnstile), and β€” with your consent β€” to measure usage and show ads. See Privacy.