hugging face

AI

FSM

The FSM can change from one state to another in response to some inputs; the change from one state to another is called a transition.

hugging face

download model

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
# option 1 
brew install git-lfs
git lfs install
git clone https://huggingface.co/<model-id> # e.g: git clone git@hf.co:bigscience/bloom

# option 2: huggingface-cli is old command cli, hf command is newer
pip install huggingface_hub
or 
# install hf
python3 -m pip install -i https://pypi.mirrors.ustc.edu.cn/simple "huggingface_hub[cli]"

# optional login
hf auth login
# or using an environment variable: hf auth login --token $HUGGINGFACE_TOKEN

# download, default download model path is ~/.cache/huggingface/hub/
hf download ByteDance-Seed/Seed-X-PPO-7B # --local-dir /path/to/custom/path

# Tells the CLI to completely ignore any existing cached files or previous partial downloads, forcing it to fetch the files freshly from the Hugging Face servers again.
--force-download

# safely deletes models
hf delete-cache

# check download cache
hf cache scan

check

parquet

apt update
apt install python3-pip
pip install –upgrade pyarrow datasets

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
import glob
import sys
import pyarrow.parquet as pq

dataset_path = "/path/to/database"
parquet_files = glob.glob(f"{dataset_path}/*.parquet")

corrupt_files = []

print(f"Checking {len(parquet_files)} parquet files...")
for file in sorted(parquet_files):
    try:
        # read_table forces PyArrow to read and decompress all row groups
        pq.read_table(file)
        print(f"✅ OK: {file.split('/')[-1]}")
    except Exception as e:
        print(f"❌ CORRUPT: {file.split('/')[-1]}")
        print(f"   Error: {e}")
        corrupt_files.append(file)

print("-" * 40)
if corrupt_files:
    print(f"🚨 Found {len(corrupt_files)} corrupt file(s). Please redownload:")
    for f in corrupt_files:
        print(f"   - {f}")
    sys.exit(1)
else:
    print("✅ All parquet files are healthy!")

safetensors

apt update

apt install python3-pip

pip install safetensors

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
import sys
from safetensors import safe_open

def check_safetensors_basic(path):
    print(f"Checking {path}...")
    try:
        # Opening it forces the safetensors library to parse the JSON header 
        # and validate the byte offsets of the tensors.
        with safe_open(path, framework="pt") as f:
            keys = list(f.keys())
            print(f"✅ safetensors file structure is healthy.")
            print(f"✅ Found {len(keys)} tensors.")
            return True
    except Exception as e:
        print("❌ safetensors file is corrupted or incomplete.")
        print("Error details:", e)
        return False

if __name__ == "__main__":
    if len(sys.argv) > 1:
        check_safetensors_basic(sys.argv[1])
    else:
        print("Usage: python check_safetensors.py <file_path>")

reference