noobaa

noobaa

Terminology

NSFS

NSFS (short for Namespace-Filesystem) is a capability to use a shared filesystem (mounted in the endpoints) for the storage of S3 buckets, while keeping a 1-1 mapping between Object and File.

reference

NSFS on Kubernetes

  • FS backend supported types are GPFS, CEPH_FS, NFSv4 default is POSIX

code

image

1
2
3
4
5
6
7
8
9
10
11
make noobaa  # include: 1 make build, 2 make base, 3 make noobaa

# include below operate
# 1. make noobaa-builder image 
docker build   --build-arg CENTOS_VER=9 --build-arg BUILD_S3SELECT=1 --build-arg BUILD_S3SELECT_PARQUET=0 -f src/deploy/NVA_build/builder.Dockerfile   -t noobaa-builder .

# 2. make noobaa-base image
docker build   --build-arg BUILD_S3SELECT=1 --build-arg BUILD_S3SELECT_PARQUET=0 -f src/deploy/NVA_build/Base.Dockerfile   -t noobaa-base .

# 3. make noobaa-core image
docker build   --build-arg CENTOS_VER=9 --build-arg BUILD_S3SELECT=1 --build-arg BUILD_S3SELECT_PARQUET=0 -f src/deploy/NVA_build/NooBaa.Dockerfile   -t noobaa --build-arg GIT_COMMIT="d6feb0a" .

quay archifact

1
2
docker pull quay.io/noobaa/noobaa-builder:master-20250623
docker pull quay.io/noobaa/noobaa-base:master-20250623

workflow

  • build image

    1
    .github/workflows/manual-full-build.yaml

Deploy

1
2
3
4
curl -LO https://github.com/noobaa/noobaa-operator/releases/download/v5.18.4/noobaa-operator-v5.18.4-linux-amd64.tar.gz
tar -xzvf noobaa-operator-v5.18.4-linux-amd64.tar.gz
chmod +x noobaa-$OS-$VERSION
mv noobaa-$OS-$VERSION /usr/local/bin/noobaa

install

  • prepare

    1
    2
    3
    sudo ctr -n k8s.io images pull dockerproxy.zetyun.cn/docker.io/noobaa/noobaa-core:master-20250623
    sudo ctr -n k8s.io images pull dockerproxy.zetyun.cn/docker.io/noobaa/noobaa-operator:5.18.4
    sudo ctr -n k8s.io images pull quay.io/sclorg/postgresql-15-c9s:latest

  • install

    1
    2
    3
    4
    5
    6
    7
    8
    kubectl create ns noobaa
    kubectl config set-context --current --namespace noobaa

    # use internal postgres
    noobaa install --noobaa-image=dockerproxy.zetyun.cn/docker.io/noobaa/noobaa-core:master-20250623 --operator-image=dockerproxy.zetyun.cn/docker.io/noobaa/noobaa-operator:5.18.4 --db-image=quay.io/sclorg/postgresql-15-c9s:latest --namespace=noobaa

    # use external postgres 
    noobaa install --noobaa-image=dockerproxy.zetyun.cn/docker.io/noobaa/noobaa-core:master-20250623 --operator-image=dockerproxy.zetyun.cn/docker.io/noobaa/noobaa-operator:5.18.4 --postgres-url="postgres://postgres:noobaa123@10.220.32.18:5432/nbcore" --namespace=noobaa

  • status

    1
    noobaa status --show-secrets

  • uninstall

    1
    noobaa uninstall --cleanup

  • upgrade

    1
    2
    3
    4
    5
    6
    7
    8
    noobaa upgrade --noobaa-image <noobaa-image-path-and-tag> --operator-image <operator-image-path-and-tag>

    # image update
    sudo ctr -n k8s.io images pull harbor.zetyun.cn/dingofs/noobaa-core:dingofs-v1.0

    # e.g.
    noobaa upgrade --noobaa-image harbor.zetyun.cn/dingofs/noobaa-core:dingofs-v1.0 --operator-image dockerproxy.zetyun.cn/docker.io/noobaa/noobaa-operator:5.18.4

pod

1
2
3
4
5
noobaa-core-0                                      2/2     Running   0          2m5s
noobaa-db-pg-0                                     1/1     Running   0          2m6s
noobaa-default-backing-store-noobaa-pod-cb1747eb   1/1     Running   0          19s
noobaa-endpoint-79677c7dd9-cjqdj                   1/1     Running   0          42s
noobaa-operator-7d969db69c-gmlm4                   1/1     Running   0          2m28s

endpoint

boostrap

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
# /noobaa_init_files/noobaa_init.sh init_endpoint
init_endpoint() {
  fix_non_root_user

  # nsfs folder is a root folder of mount points to backing storages.
  # In oder to avoid access denied of sub folders, configure nsfs with full permisions (777)
  if [ -d "/nsfs" ];then
    chmod 777 /nsfs
  fi

  cd /root/node_modules/noobaa-core/
  run_internal_process node --unhandled-rejections=warn ./src/s3/s3rver_starter.js
}

run_internal_process() {
  while true
  do
    local package_path="/root/node_modules/noobaa-core/package.json"
    local version=$(cat ${package_path} | grep version | awk '{print $2}' | sed 's/[",]//g')
    echo "Version is: ${version}"
    echo "Running: $*"
    $*
    rc=$?
    echo -e "\n\n\n"
    echo "######################################################################"
    echo "$(date) NooBaa: Process exited RIP (RC=$rc)"
    echo "######################################################################"
    echo -e "\n\n\n"

    mode="manual" # initial value just to start the loop
    while [ "$mode" == "manual" ]
    do
      # load mode from file/env
      if [ -f "./NOOBAA_INIT_MODE" ]
      then
        mode="$(cat ./NOOBAA_INIT_MODE)"
      else
        mode="$NOOBAA_INIT_MODE"
      fi

      if [ "$mode" == "auto" ]
      then
        echo "######################################################################"
        echo "$(date) NooBaa: Restarting process (NOOBAA_INIT_MODE=auto)"
        echo "######################################################################"
        echo -e "\n\n\n"
        # will break from the inner loop and re-run the process
      elif [ "$mode" == "manual" ]
      then
        echo "######################################################################"
        echo "$(date) NooBaa: Waiting for manual intervention (NOOBAA_INIT_MODE=manual)"
        echo "######################################################################"
        echo -e "\n"
        sleep 10
        # will re-enter the inner loop and reload the mode
      else
        [ ! -z "$mode" ] && echo "NooBaa: unrecognized NOOBAA_INIT_MODE = $mode"
        return $rc
      fi
    done
  done
}

noobaa-db-pg-0

1
2
3
4
5
6
# volume
/var/lib/pgsql from db (rw)
db:
  Type:       PersistentVolumeClaim (a reference to a PersistentVolumeClaim in the same namespace)
  ClaimName:  db-noobaa-db-pg-0
  ReadOnly:   false

serivice

1
2
3
4
5
6
NAME            TYPE           CLUSTER-IP      EXTERNAL-IP   PORT(S)                                                    AGE
noobaa-db-pg    ClusterIP      None            <none>        5432/TCP                                                   2m15s
noobaa-mgmt     ClusterIP      10.97.139.113   <none>        80/TCP,443/TCP,8445/TCP,8446/TCP                           2m14s
noobaa-syslog   ClusterIP      10.102.80.226   <none>        514/UDP                                                    2m11s
s3              LoadBalancer   10.110.97.124   <pending>     80:32292/TCP,443:32034/TCP,8444:31155/TCP,7004:30222/TCP   2m14s
sts             LoadBalancer   10.103.219.22   <pending>     443:30126/TCP                                              2m13s

volume

1
2
3
4
5
6
noobaa-db-noobaa-db-pg-0
noobaa-noobaa-default-backing-store-noobaa-pvc-07c805ab

# mountpod 
dingofs-ubuntu2-pvc-ef31f21c-5677-4f83-94ad-face89b741c3-lhenuc
dingofs-ubuntu3-pvc-10fd093a-0d75-45ee-b52d-84f712378e09-jbxeue

systemc info

1
2
3
4
5
6
7
8
9
10
11
12
13
14
NOOBAA_SECRET=$(kubectl get noobaa noobaa -n noobaa -o json | jq -r '.status.accounts.admin.secretRef.name' )
noobaa-admin

NOOBAA_MGMT=$(kubectl get noobaa noobaa -n noobaa -o json | jq -r '.status.services.serviceMgmt.nodePorts[0]' )
https://10.xx.xx.18:0

NOOBAA_S3=$(kubectl get noobaa noobaa -n noobaa -o json | jq -r '.status.services.serviceS3.nodePorts[0]' )
https://10.xx.xx.18:30478   # entrypoint

NOOBAA_ACCESS_KEY=$(kubectl get secret noobaa-admin -n noobaa -o json | jq -r '.data.AWS_ACCESS_KEY_ID|@base64d')
3gR59xxxxxHBbOdOx

NOOBAA_SECRET_KEY=$(kubectl get secret noobaa-admin -n noobaa -o json | jq -r '.data.AWS_SECRET_ACCESS_KEY|@base64d')
zu229xxxxxxxxxxxxxxxxxxxxxxxxcfX

Mgmt UI

1
2
3
4
5
6
7
8
9
10
11
kubectl get secret noobaa-admin -n noobaa -o json | jq '.data|map_values(@base64d)'
# print 
{
  "AWS_ACCESS_KEY_ID": "3gR59eoLoBvhjHBbOdOx",
  "AWS_SECRET_ACCESS_KEY": "zu229aAJIo6g9GOsjKTyUc6p3Rc0c/nHGdjI3cfX",
  "email": "admin@noobaa.io",
  "password": "7JfrgTXgnJjU4ywSFWYr+Q==",
  "system": "noobaa"
}

open $NOOBAA_MGMT

aws-cli

1
2
3
4
alias s3='AWS_ACCESS_KEY_ID=3gR5xxxxxxxxOx AWS_SECRET_ACCESS_KEY=zu229aAJxxxxxxxxxxxxxxxxxcfX aws --endpoint https://10.xxx.xx.18:30478 --no-verify-ssl s3'
s3 ls
s3 sync /var/log/ s3://first.bucket
s3 ls s3://first.bucket

NSFS

1. create NSFS resource

1
noobaa namespacestore create nsfs dingofs-nsfs --pvc-name='noobaa-nsfs-pvc'

dynamic-pvc-1.yaml

1
2
3
4
5
6
7
8
9
10
11
12
13
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: noobaa-nsfs-pvc
  namespace: noobaa
spec:
  accessModes:
  - ReadWriteMany
  resources:
    requests:
      storage: 100Gi
  volumeMode: Filesystem
  storageClassName: dingofs-sc-s3

delete

1
noobaa namespacestore delete dingofs-nsfs

2. create bucket

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
noobaa api bucket_api create_bucket '{
  "name": "dingofs-bucket-1",
  "namespace":{
    "write_resource": { "resource": "dingofs-nsfs", "path": "dingofs-bucket-1/" },
    "read_resources": [ { "resource": "dingofs-nsfs", "path": "dingofs-bucket-1/" }]
  }
}'

# log
bucket_type: NAMESPACE
data:
  available_quantity_for_upload:
    "n": 1125899906842623
    peta: 7
  available_size_for_upload: 0
  free: 0
  last_update: 1750931642170
  size: 0
  size_reduced: 0
mode: OPTIMAL
name: dingofs-bucket-1
namespace:
  read_resources:
  - path: dingofs-bucket-1/
    resource: dingofs-nsfs
  should_create_underlying_storage: false
  write_resource:
    path: dingofs-bucket-1/
    resource: dingofs-nsfs
num_objects:
  last_update: 1750931550000
  value: 0
owner_account:
  email: operator@noobaa.io
  id: 685d128d454ec60022bf1918
stats_by_type: []
storage:
  last_update: 1750931642170
  values:
    free: 0
    total: 0
    used: 0
    used_other: 0
tag: ""
usage_by_pool:
  last_update: 1750931550000
  pools: []
versioning: DISABLED
writable: false

status

1
noobaa bucket status <bucketName>

list bucket

1
2
3
noobaa api bucket_api list_buckets '{}'
or 
noobaa bucket list

get_bucket_policy

1
noobaa api bucket_api get_bucket_policy '{"name": "<bucketName>"}'

delete bucket

1
2
3
noobaa api bucket_api delete_bucket '{"name": "<bucketName>"}'
or 
noobaa bucket delete <bucketName>

3. add bucket policy

1
2
# hould update the bucket policy using admin account
AWS_ACCESS_KEY_ID=3gR59eoxxxxBbOdOx AWS_SECRET_ACCESS_KEY=zu229aAJIxxxxxxxxxxxdjI3cfX aws --endpoint-url=https://10.220.32.18:30478 --no-verify-ssl s3api put-bucket-policy --bucket dingofs-bucket-1 --policy file://policy.json

policy.json

1
2
3
4
5
6
7
8
9
10
11
12
{
"Version":"2025-06-25",
"Statement":[
        {
        "Sid":"id-1",
        "Effect":"Allow",
        "Principal":"*",
        "Action":["s3:*"],
        "Resource":["arn:aws:s3:::*"]
        }
]
}

Sid (Statement ID) serves as an optional identifier for individual policy statements, allowing for easier management and referencing of specific rules within a larger policy. It’s a way to give a unique name to each permission set within the bucket policy, which can be helpful when dealing with multiple statements or when debugging policy issues

4. create account

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
noobaa api account_api create_account '{
    "email": "dingofs@zetyun.com",
    "name": "dingofs",
    "has_login": false,
    "s3_access": true,
    "default_resource": "dingofs-nsfs",
    "nsfs_account_config": {
      "uid": 0,
      "gid": 0,
      "new_buckets_path": "/",
      "nsfs_only": false
     }
  }'
  
# print 
access_keys:
access_key: UanhXoxxxxxIggJP
secret_key: ptsSJUYxxxxxxxx2SF8ltV
token: eyJhbGcixxxxxxxxxxxxxxxxxxxxxxxxxxxxxqGdB-E2zQF-3MBII
  • check account
    1
    2
    3
    noobaa api account_api list_accounts {}
    # check specify accout
    noobaa api account_api read_account '{"email":"jenia@noobaa.io"}'

5. config s3 client

1
alias s3-user-dingofs='AWS_ACCESS_KEY_ID=UanhxxxxxxggJP AWS_SECRET_ACCESS_KEY=ptsSJUYVCxxxxxxxxxxxSF8ltV aws --endpoint https://10.220.32.18:30478  --no-verify-ssl s3'

6. operate

  • create bucket
    s3-user-datacanvas mb s3://test-bucket

Best Practices

External Postgresql DB (TBD)

  • binary install

    1
    2
    3
    4
    5
    6
    sudo dnf install -y https://download.postgresql.org/pub/repos/yum/reporpms/EL-9-x86_64/pgdg-redhat-repo-latest.noarch.rpm
    sudo dnf -qy module disable postgresql
    sudo dnf install -y postgresql17-server
    sudo /usr/pgsql-17/bin/postgresql-17-setup initdb
    sudo systemctl enable postgresql-17
    sudo systemctl start postgresql-17

  • docker install

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    # 17 version
    docker run --name noobaa-postgres \
      -e POSTGRES_PASSWORD=<> \
      --restart always \
      --network host \
      -d dockerproxy.zetyun.cn/docker.io/postgres:latest
      
    # 15 version
    docker run --name noobaa-postgres \
      -e POSTGRESQL_ADMIN_PASSWORD=<> \
      --restart always \
      --network host \
      -d quay.io/sclorg/postgresql-15-c9s:latest

    # enter psql
    psql -U postgres

    # create nbcore database
    CREATE DATABASE nbcore WITH LC_COLLATE = 'C' TEMPLATE template0;

    # check 
    \list

    # db url 
    postgres://postgres:<mysecretpassword>@<ip>:5432/nbcore"

    use postgres 15, should config

    variables:

    POSTGRESQL_USER POSTGRESQL_PASSWORD POSTGRESQL_DATABASE

    Or the following environment variable:

    POSTGRESQL_ADMIN_PASSWORD

    Or both.

Troubleshooting