Raspberry Piクラスタにおける軽量LLMのKubernetes Pod化と推論

$ sudo apt update
$ sudo apt install -y build-essential cmake libcurl4-openssl-dev
$ git clone https://github.com/ggerganov/llama.cpp
$ cd llama.cpp
$ mkdir build && cd build
$ cmake ..
$ make -j4

$ mkdir -p ~/models/phi3 && cd ~/models/phi3
$ wget https://huggingface.co/microsoft/Phi-3-mini-4k-instruct-gguf/resolve/main/Phi-3-mini-4k-instruct-q4.gguf

$ cd ~/llama.cpp/
$ ./build/bin/llama-cli -m ~/models/phi3/Phi-3-mini-4k-instruct-q4.gguf -p "日本で2番目に高い山は？" -n 128

$ ./build/bin/llama-server -m ~/models/phi3/Phi-3-mini-4k-instruct-q4.gguf -c 2048 -n 128 --host 0.0.0.0 --port 8080

# 事前準備
$ sudo apt update
$ sudo apt install -y ca-certificates curl gnupg
# Docker公式GPGキーを追加
$ sudo install -m 0755 -d /etc/apt/keyrings
$ curl -fsSL https://download.docker.com/linux/debian/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg
# DockerのAPTリポジトリを追加
$ echo \
  "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \
  https://download.docker.com/linux/debian \
  $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \
  sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
# Dockerエンジンをインストール
$ sudo apt update
$ sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
# ユーザーを docker グループに追加（sudoなしで使いたい場合）
$ sudo usermod -aG docker $USER
# このあと 一度ログアウトして再ログイン してください（グループ変更を反映させるため）もしくは次のコマンドでセッションを更新
$ newgrp docker
# バージョン確認
$ docker --version
Docker version 28.1.1, build 4eba377

$ docker build -t llama-server:0.1.0 .

$ docker images
REPOSITORY     TAG       IMAGE ID       CREATED          SIZE
llama-server   0.1.0     3b1f7b166060   24 seconds ago   833MB

# registryの永続化ボリュームを指定する場合
docker run -d -p 5000:5000 --restart=always --name registry \
  -v /opt/registry:/var/lib/registry \
  registry:2

sudo nano /etc/docker/daemon.json
# /etc/docker/daemon.json に以下を追加：なければファイル作成
{
  "insecure-registries": ["localhost:5000"]
}

# デーモン再起動
sudo systemctl restart docker

$ sudo nmcli con mod "Wired connection 1" \
  ipv4.addresses 192.168.0.161/24 \
  ipv4.gateway 192.168.0.1 \
  ipv4.dns "192.168.0.1 8.8.8.8" \
  ipv4.method manual

# 設定を反映（再起動または接続の再アクティベート）：

$ sudo nmcli con down "Wired connection 1" && sudo nmcli con up "Wired connection 1"

$ sudo nano /etc/hosts

$ sudo hostnamectl set-hostname raspi-ctrl

# raspi-ctrl.cnf
[req]
default_bits       = 4096
prompt             = no
default_md         = sha256
distinguished_name = dn
x509_extensions    = v3_req

[dn]
CN = raspi-ctrl.local

[v3_req]
subjectAltName = @alt_names

[alt_names]
DNS.1 = raspi-ctrl.local

$ mkdir -p ~/certs && cd ~/certs 
$ openssl req -x509 -nodes -days 365 \
  -newkey rsa:4096 \
  -keyout raspi-ctrl.key \
  -out raspi-ctrl.crt \
  -config raspi-ctrl.cnf
  
# 証明書ファイルの読み取り権限を修正
$ chmod 644 ~/certs/domain.key

# 一旦元のレジストリは削除
$ docker stop registry
registry
$ docker rm registry
registry

# Registry の起動（HTTPS化）
docker run -d -p 443:5000 --restart=always --name registry \
  -v /opt/registry:/var/lib/registry \
  -v /etc/ssl/private/raspi-ctrl.crt:/certs/domain.crt \
  -v /etc/ssl/private/raspi-ctrl.key:/certs/domain.key \
  -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt \
  -e REGISTRY_HTTP_TLS_KEY=/certs/domain.key \
  registry:2

[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
  [plugins."io.containerd.grpc.v1.cri".registry.mirrors."raspi-ctrl.local"]
    endpoint = ["https://raspi-ctrl.local"]

sudo systemctl restart containerd

# タグ付け
$ docker tag llama-server:0.1.0 raspi-ctrl.local/llama-server:0.1.0
# レジストリにPush
$ docker push raspi-ctrl.local/llama-server:0.1.0
The push refers to repository [raspi-ctrl.local/llama-server]
150643f5166c: Pushed 
ef16af6c922e: Pushed 
5f70bf18a086: Pushed 
401146b66e2c: Pushed 
60e6f28b09b9: Pushed 
fec0f5f70886: Pushed 
66a307619b8c: Pushed 
0.1.0: digest: sha256:4767519047ea7e9714cec1d055f60a8cb3ae2033d8ab3a606ec46155dee602ee size: 1786
emboss@raspi-ctrl:~/certs $ 

$ kubectl apply -f llm-ns.yaml
# 確認
$ kubectl get namespaces
NAME              STATUS   AGE
default           Active   30h
kube-flannel      Active   30h
kube-node-lease   Active   30h
kube-public       Active   30h
kube-system       Active   30h
llm               Active   36s
longhorn-system   Active   24h

$ kubectl config set-context --current --namespace=llm

# phi3-model-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: phi3-model-pvc
  namespace: llm
spec:
  accessModes:
    - ReadWriteMany
  resources:
    requests:
      storage: 3Gi
  storageClassName: nfs-client

$ kubectl apply -f phi3-model-pvc.yaml 
persistentvolumeclaim/phi3-model-pvc created

# 確認
$ kubectl get PersistentVolumeClaim
NAME             STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   VOLUMEATTRIBUTESCLASS   AGE
phi3-model-pvc   Bound    pvc-04e910a0-b856-4226-9741-1842c50805a2   3Gi        RWX            nfs-client     <unset>                 6s
emboss@raspi-ctrl:~/dev/edgeaiagent/ai/llama-server/k8s $ 

$ cp ~/models/phi3/Phi-3-mini-4k-instruct-q4.gguf /nfs-share/llm-phi3-model-pvc-pvc-04e910a0-b856-4226-9741-1842c50805a2/

$ ls -l /nfs-share/llm-phi3-model-pvc-pvc-04e910a0-b856-4226-9741-1842c50805a2/
total 2337144
-rw-r--r-- 1 emboss emboss 2393231072 May  2 14:00 Phi-3-mini-4k-instruct-q4.gguf

apiVersion: apps/v1
kind: Deployment
metadata:
  name: llama-server
  namespace: llm
spec:
  replicas: 1
  selector:
    matchLabels:
      app: llama-server
  template:
    metadata:
      labels:
        app: llama-server
    spec:
      containers:
        - name: llama-server
          image: raspi-ctrl.local/llama-server:0.1.0
          ports:
            - containerPort: 8080
          volumeMounts:
            - name: model-volume
              mountPath: /models
      volumes:
        - name: model-volume
          persistentVolumeClaim:
            claimName: phi3-model-pvc

apiVersion: v1
kind: Service
metadata:
  name: llama-service
  namespace: llm
spec:
  type: NodePort
  selector:
    app: llama-server
  ports:
    - port: 8080
      targetPort: 8080
      nodePort: 30080