• Jenkins
• Setup

CI.FreeBSD.org Setup

jails

jenkins.ci.freebsd.org

Packages

jenkins
git-tiny

/etc/rc.conf

local_unbound_enable="YES"

jenkins_enable="YES"
jenkins_home="/usr/local/jenkins"
jenkins_args="--webroot=${jenkins_home}/war --httpPort=8180"
jenkins_java_opts="-Djavax.net.ssl.trustStore=/usr/local/jenkins/cacerts -Djava.net.preferIPv6Addresses=true"

nginx.ci.freebsd.org

Packages

nginx

nginx.conf

worker_processes  1;
events {
    worker_connections  1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  /var/log/nginx-access.log  main;
    sendfile        on;
    keepalive_timeout  65;
    server {
        listen       80;
        listen       [::]:80;
        server_name  ci.freebsd.org;
        client_max_body_size 10M;
        client_body_buffer_size 128k;
        location ~ /(doFingerprintCheck|fingerprintCheck) {
            deny all;
            return 403;
        }
        location /tinderbox {
            alias  /usr/local/www/tinderbox/jenkins-tinderbox;
            index  index.html;
        }
        location /robots.txt {
            alias /usr/local/www/ci.freebsd.org/robots.txt;
        }
        location /FreeBSD_logo.png {
            alias /usr/local/www/ci.freebsd.org/FreeBSD_logo.png;
        }
        location /favicon.ico {
            alias /usr/local/www/ci.freebsd.org/favicon.ico;
        }
        # maintenance page
#        location / {
#            root   /usr/local/www/ci.freebsd.org;
#            index  index.html;
#        }
        location / {
            proxy_pass              http://jenkins.ci.freebsd.org:8180;

            proxy_set_header        Host $host;
            proxy_set_header        X-Real-IP $remote_addr;
            proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header        X-Forwarded-Proto https;

            proxy_redirect          http:// https://;

            proxy_connect_timeout   300;
            proxy_send_timeout      300;
            proxy_read_timeout      300;
            send_timeout            300;

            rewrite                 ^/jenkins/?(.*)$ /$1 permanent;
        }

    }
}

/usr/local/www/ci.freebsd.org/robots.txt

# we don't want robots to click "build" links
User-agent: *
Disallow: /

artifact.ci.freebsd.org

Packages

nginx
pure-ftpd

on jail host

zfs create zroot/artifact

/etc/fstab.artifact

/artifact /j/jails/artifact/home/artifact nullfs rw 0 0

user/group

artifact:*:1000:1000::0:0:artifact owner:/home/artifact:/usr/sbin/nologin

artifact:*:1000:

/etc/rc.conf

local_unbound_enable="YES"

pureftpd_enable="YES"
nginx_enable="YES"

pure-ftpd.conf

ChrootEveryone              yes
BrokenClientsCompatibility  no
MaxClientsNumber            50
Daemonize                   yes
MaxClientsPerIP             4
VerboseLog                  no
DisplayDotFiles             yes
AnonymousOnly               no
NoAnonymous                 yes
SyslogFacility              ftp
DontResolve                 yes
MaxIdleTime                 1
PureDB                      /usr/local/etc/pureftpd.pdb
LimitRecursion              10000 8
AnonymousCanCreateDirs      no
MaxLoad                     4
PassivePortRange            30000 32000
AntiWarez                   no
Umask                       133:022
MinUID                      100
AllowUserFXP                no
AllowAnonymousFXP           no
ProhibitDotFilesWrite       no
ProhibitDotFilesRead        no
AutoRename                  no
AnonymousCantUpload         no
MaxDiskUsage                99
CustomerProof               yes
TLS                         2
CertFile                    /usr/local/etc/artifact.ci.FreeBSD.org.pem

pure-ftpd user

pure-pw useradd pure-pw mkdb

nginx.conf

worker_processes  4;
events {
    worker_connections  1024;
}
http {
    include       mime.types;
    default_type  application/octet-stream;
    sendfile        on;
    keepalive_timeout  65;

    server_tokens off;

    ssl_prefer_server_ciphers on;
    ssl_ciphers "ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA
    ssl_session_cache shared:SSL:64m;
    ssl_session_timeout 10m;
    ssl_session_tickets off;
    ssl_protocols TLSv1 TLSv1.1 TLSv1.2;

    ssl_stapling on;
    ssl_stapling_verify on;
    ssl_trusted_certificate /usr/local/share/certs/ca-root-nss.crt;
    resolver_timeout 10s;

    server {
        listen       80;
        listen       [::]:80;
        server_name  artifact.ci.freebsd.org;
        add_header   Strict-Transport-Security "max-age=31536000; includeSubDomains";
        return       301 https://$server_name$request_uri;

        ssl_certificate      /etc/clusteradm/acme-certs/artifact.ci.freebsd.org.crt;
        ssl_certificate_key  /etc/ssl/artifact.ci.freebsd.org.key;
    }

    server {
        listen       80;
        listen       [::]:80;
        listen       443 ssl;
        listen       [::]:443 ssl;
        server_name  artifacts.ci.freebsd.org;
        add_header   Strict-Transport-Security "max-age=31536000; includeSubDomains";
        return       301 https://artifact.ci.freebsd.org$request_uri;

        ssl_certificate      /etc/clusteradm/acme-certs/artifact.ci.freebsd.org.crt;
        ssl_certificate_key  /etc/ssl/artifact.ci.freebsd.org.key;
    }

    server {
        listen       443 ssl;
        listen       [::]:443 ssl;
        server_name  artifact.ci.freebsd.org;
        add_header   Strict-Transport-Security "max-age=31536000; includeSubDomains";

        ssl_certificate      /etc/clusteradm/acme-certs/artifact.ci.freebsd.org.crt;
        ssl_certificate_key  /etc/ssl/artifact.ci.freebsd.org.key;

        location / {
            root   /home/artifact;
            fancyindex on;
            fancyindex_name_length 100;
            index  index.html index.htm;
        }
        error_page   500 502 503 504  /50x.html;
        location = /50x.html {
            root   /usr/local/www/nginx-dist;
        }
    }

    server {
        listen       8182 ssl;
        listen       [::]:8182 ssl;
        server_name  localhost;
        server_name  artifact.ci.freebsd.org;
        add_header   Strict-Transport-Security "max-age=31536000; includeSubDomains";

        ssl_certificate      /etc/clusteradm/acme-certs/artifact.ci.freebsd.org.crt;
        ssl_certificate_key  /etc/ssl/artifact.ci.freebsd.org.key;

        access_log   /var/log/artifact/nginx-access.log  main;
        error_log    /var/log/artifact/nginx-error.log;

        charset UTF-8;

        location / {
            proxy_pass   http://127.0.0.1:4080/;
        }
    }
}

admin.ci.freebsd.org

Packages

py37-jenkins-job-builder

Jenkins Master

Main configure

Manage Jenkins -> Configure System

• Subversion

  • Subversion Workspace Version -> 1.8

plugin

(not necessary if your IPv4 routing works)

HTTP Proxy Configuration

Server: proxy.nyi.FreeBSD.org
Port: 3128

Update Site

https://updates.jenkins.io/update-center.json

Install plugin list:

• https://plugins.jenkins.io/build-timeout/

• https://plugins.jenkins.io/checkstyle/

• https://plugins.jenkins.io/clang-scanbuild/

• https://plugins.jenkins.io/groovy/

• https://plugins.jenkins.io/leastload/

• https://plugins.jenkins.io/postbuildscript/

• https://plugins.jenkins.io/preSCMbuildstep/

• https://plugins.jenkins.io/publish-over-ftp/

• https://plugins.jenkins.io/timestamper/

• https://plugins.jenkins.io/warnings/

Auth

Jenkins/LDAP

jenkins job builder

https://github.com/freebsd/freebsd-ci/tree/master/jjb

configuration backup

refer to Jenkins/Backup, but we put repository at /home/jenkins-config

Jenkins Build Node (agent)

Base

zfs set compression=lz4 zroot

Packages

curl
expect
git-tiny
jq
openjdk17
python3
sudo

Configure

# On agent machine
# assmue we have a "zroot" zpool, with mountpoint=/
# if no, add "-o mountpoint=/jenkins" after "zfs create"
zfs create zroot/jenkins
zfs create zroot/jenkins/jails
zfs create zroot/jenkins/workspace

# create "jenkins" group
# create "jenkins" with /jenkins as home and /bin/sh as shell
pw group add jenkins -g 5213
pw useradd jenkins -d '/jenkins' -w no -u 5213 -g 5213 -s '/bin/sh' -c 'Jenkins CI'

chown jenkins:jenkins /jenkins /jenkins/workspace

zfs snapshot zroot/jenkins/jails@empty
zfs snapshot zroot/jenkins/workspace@empty

# On Jenkins UI
# Manage Jenkins -> Manage Nodes -> New Node

Node name: (hostname)
Select "Permanent Agent"

Remote root directory: /jenkins
Labels: jailer (based on your job definition)
Usage: Use this node as much as possible (based on your need)
Launch method:
  Launch agent by connecting it to the master
  Advanced -> Tunnel connection through: jenkins.ci.freebsd.org:8181

# Environment variables:
BUILDER_0_IP6 (V6 IP for disposable jail.)
BUILDER_HTTP_PROXY http://proxy.nyi.FreeBSD.org:3128
BUILDER_JFLAG 12 (based on your kern.smp.cpus)
BUILDER_NETIF igb0 (outgoing interface for jail/vm)
BUILDER_RESOLV_CONF nameserver 2610:1c1:1:6002::100\nnameserver 2610:1c1:1:6002::200\n
BUILDER_ZFS_PARENT zroot/jenkins/jails

# On agent machine
su - jenkins

git config --global http.proxy http://proxy.nyi.FreeBSD.org:3128
git clone https://github.com/lwhsu/jenkins-agent-scripts
cd jenkins-agent-scripts

cp agent.conf.sample agent.conf
# go to https://${master}/computer/${hostname}/
# get ${master} and ${secret} from "Run from agent command line:"
# java -jar agent.jar -jnlpUrl https://${master}/computer/${hostname}/agent-agent.jnlp -secret ${secret}
# update agent.conf

crontab crontab

/usr/local/etc/sudoers.d/jenkins

Cmnd_Alias CI_COMMANDS = /usr/sbin/jail, /usr/sbin/jexec, /sbin/mount, /sbin/umount, /sbin/devfs, /bin/chflags, /bin/rm, /usr/sbin/pkg, /usr/bin/tar, /sbin/ifconfig, /usr/bin/tee, /sbin/zfs, /bin/mkdir, /sbin/mdconfig, /usr/sbin/bhyvectl, /usr/sbin/bhyveload, /usr/sbin/bhyve, /usr/bin/timeout -k [0-9]* [0-9]* /usr/sbin/bhyve *
Defaults:jenkins !env_reset
jenkins ALL=(root) NOPASSWD: CI_COMMANDS

/boot/loader.conf

vmm_load="YES"

/etc/rc.conf

cloned_interfaces="bridge0 tap0 tap1" # The number of tap* depends on how many worker slots
autobridge_interfaces="bridge0"
autobridge_bridge0="igb0 tap*" # the first argument is the outgoing interface
ifconfig_bridge0="up"

/etc/sysctl.conf

net.link.tap.up_on_open=1

Setup git repository mirror at /home/git

freebsd.org only

• /etc/ssh/sshd_config.listen
• /etc/sysctl.conf
  • vfs.zfs.arc_max
  • vfs.zfs.arc_free_target

---

CategoryServices