/etc/systemd/system - default location for system units ~/.config/systemd/user - default location for user units


Flags for operating on user units are given in []

# reload units and timers
systemctl [--user] daemon-reload
# show all units (including disabled)
systemctl [--user] list-units -a
# view logs for unit
# also accepts:
#   -f               | tail the log
#   --user-unit foo  | target user unit instead of system
#   --boot=0         | show logs from current boot (-1 for previous, etc)
journalctl --unit foo


A simple service unit


 Description=example service



foobar.timer A simple timer unit

 Description=example timer

 # run every 15 minutes (aligns to the hour)
 # run timer immediately if script is enabled and is past due


Basic Arch Install


timedatectl set-ntp true

fdisk /dev/sda 

# Create 300MB boot, 2GB swap, and leave the rest for root

mkswp /dev/sda2

mkfs.ext4 /dev/sda3

mount /dev/sda3 /mnt

swapon /dev/sda2

# edit /etc/pacman.d/mirrorlist to change mirror order **

pacstrap /mnt base

genfstab -p /mnt >> /mnt/etc/fstab

arch-chroot /mnt

ln -s /usr/share/zoneinfo/America/Indianapolis /etc/localtime

hwclock --systohc --utc

# uncomment en_US locales in /etc/locale.gen **


# enter hostname in /etc/hostname **

mkinitcpio -p linux


pacman -S grub

grub-install --target=i386-pc --recheck --debug /dev/sda

grub-mkconfig -o /boot/grub/grub.cfg



pacman -S vim htop git openssh wget

pacman -S xorg-server xf86-video-ati xorg-xinit

Generic Linux Install

# Copy bootable image to flash drive (status=progress requires dd >= 8.24)
dd if=foobar.iso if=/dev/sdX status=progress && sync


# show device connect/disconnects and fired rules
udevadm monitor
# list all attributes for a particular device (and parents)
udevadm info --attribute-walk --path=/devices/...
# simple vendorid, productid example
# note SYMLINK is only for block devices (I think, look it up yourself)
SUBSYSTEMS=="usb", ATTRS{idProduct}=="3300", ATTRS{idVendor}=="1e10", MODE="0666", SYMLINK+="foobar"



# list all tables
iptables -L -n -v
# (fedora) save iptables rules and remember to disable firewalld
iptables-save > /etc/sysconfig/iptables


# allow ssh
# must allow incoming connection and response

# append rule to input (-A INPUT) on input interface enp6s0f0 (-i enp6s0f0) 
# with destination port 22 (--dport 22).  use 'state' module (-m state)
# and allow new and established connections (--state NEW,ESTABLISHED)
# jump to target ACCEPT (-j ACCEPT)
iptables -A INPUT -i enp6s0f0 -p tcp --dport 22 -m state --state NEW,ESTABLISHED -j ACCEPT

# append rule to output (-A OUTPUT) on output interface enp6s0f0 (-o enp6s0f0) 
# with source port 22 (--sport 22).  use 'state' module (-m state)
# and allow established connections (--state ESTABLISHED)
# jump to target ACCEPT (-j ACCEPT)
iptables -A OUTPUT -o enp6s0f0 -p tcp --sport 22 -m state --state ESTABLISHED -j ACCEPT
# filter table: flush all chains, and delete all user added chains
iptables -F
iptables -X
# nat table: flush all chains, and delete all user added chains
iptables -t nat -F
iptables -t nat -X



# create new lv `foo` in group `foo_group`
lvcreate -L 10G foo_group -n foo


lvremove /dev/[vgname]/[lvname]


# add all devices to filesystem
btrfs device add /dev/sdb2 /dev/sdc2 /dev/sdd2 /
# convert system to raid10
btrfs balance start -dconvert=raid10 -mconvert=raid10 /
# check balance progress
btrfs balance status /
# get rid of single chunks to get another shot at degraded,rw mount
btrfs balance start -dconvert=raid10,soft -mconvert=raid10,soft  /mount


Config examples

/etc/lxc/lxc.conf - set path for containers to be stored (default /var/lib/lxc)

lxc.lxcpath = "/lxc"

/etc/lxc/default.conf - config options for all newly created containers to inherit = veth = lxcbr0 = up = 00:16:3e:xx:xx:xx = 1

# address = =

# memory
lxc.cgroup.memory.limit_in_bytes = 512M

# memory + swap
lxc.cgroup.memory.memsw.limit_in_bytes = 1G

/etc/default/lxc-net - it may be necessary to add /etc/lxc/dnsasq.conf to the apparmor profile (/etc/apparmor.d/*dnsmasq*) with read privileges




iptables config

## Evan Widloski - 2016-11-11
# Diode iptables rules

# filter table: flush all chains, and delete all user added chains
iptables -F
iptables -X
# nat table: flush all chains, and delete all user added chains
iptables -t nat -F
iptables -t nat -X

# set default policies to DROP packets
iptables -P INPUT DROP
iptables -P OUTPUT DROP
iptables -P FORWARD DROP

# allow inbound outbound traffic on host 
iptables -A OUTPUT -o enp6s0f0 -d -j ACCEPT 
iptables -A INPUT -i enp6s0f0 -m state --state ESTABLISHED,RELATED -j ACCEPT

# set up chain for sshguard
iptables -N sshguard
iptables -A INPUT -p tcp --dport 22 -j sshguard

# allow ssh
iptables -A INPUT -i enp6s0f0 -p tcp --dport 22 -m state --state NEW,ESTABLISHED -j ACCEPT
iptables -A OUTPUT -o enp6s0f0 -p tcp --sport 22 -m state --state ESTABLISHED -j ACCEPT

# allow mosh
iptables -A INPUT -i enp6s0f0 -p udp --dport 60000:61000 -j ACCEPT
iptables -A OUTPUT -o enp6s0f0 -p udp --sport 60000:61000 -j ACCEPT

# allow connections to varnish service
#iptables -A INPUT -i enp6s0f0 -p tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT
#iptables -A OUTPUT -o enp6s0f0 -p tcp --sport 80 -m state --state ESTABLISHED -j ACCEPT

# allow host to access LXC targets via network
iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
iptables -A OUTPUT -s -j ACCEPT

# allow outbound traffic for lxc containers
iptables -A FORWARD -i lxcbr0 -j ACCEPT
iptables -t nat -A POSTROUTING -s -j MASQUERADE

# after incoming packets have been NAT'ed (see below), allow them to pass through
# the forward chain to their intended LXC target
iptables -A FORWARD -m state --state NEW,ESTABLISHED,RELATED -j ACCEPT

##------------ evan --------------
## ssh
iptables -t nat -A PREROUTING -p tcp --dport 20022 -j DNAT --to-destination


# list container statuses and ip addresses (fancy mode)
lxc-ls -f
brctl show
brctl delbr virbr0
brctl addbr virbr0
ip link set virbr0 down

New Container Setup

New LXC containers are very barebones and need a bit of setup to be useful. Here is an overview of steps for various distros.


Setup PATH

# add /bin, /sbin to path
echo 'PATH=$PATH:/bin:/sbin'>>.bashrc

Install packages

# core commands
apt-get install apt-utils vim man tar less iputils-ping

# extra commands
apt-get install git zip autojump wget htop ncdu nload


Install packages

# core commands
dnf install vim man

# core commands
dnf install git zip autojump wget htop ncdu nload


# enable notifications for any messages in buffer (works for Android client, too)
/buffer set highlight_regex .\ast{}.*


Checking state and simulating failure

 # check RAID state
 cat /proc/mdstat  # look for failure, (F), after the drive name: sda1[0](F)

 # simulate a failed drive
 mdadm --manage --set-faulty /dev/md/pv00 /dev/sda1

 # remove faulty state by removing and readding
 mdadm --remove /dev/md/pv00 /dev/sda1
 mdadm --add /dev/md/pv00 /dev/sda1

Replacing a failed drive (sdc)

# set hard drive as failed
# mark as failed and remove
mdadm --manage /dev/md127 --fail /dev/sdc1
mdadm --manage /dev/md127 --remove /dev/sdc1

# write down serial number of failed drive
hdparm -i /dev/sdc1 | grep -i serial
shutdown -h now
# remove broken harddrive, insert the new hardddrive

# copy partition scheme from working harddrive to new harddrive
sfdisk -d /dev/sda | sfdisk /dev/sdc

# add new harddrive
mdadm --manage /dev/md127 --add /dev/sdc1

# verify that array is recovering
cat /proc/mdstat

Notifying on harddrive failure (gmail)


# add this after `begin routers` in router config section
     driver = manualroute
     domains = ! +local_domains
     transport = gmail_smtp
     route_list = *
# add this after `begin transports` in transports config section
     driver = smtp
     port = 587
     hosts_require_auth =
     hosts_require_tls =
# add this after `begin authenaticators` in authentication config section
     driver = plaintext
     public_name = LOGIN
     client_send = : : password_in_plaintext_here


AUTO +imsm +1.x -all
ARRAY /dev/md/pv00 level=raid5 num-devices=4 UUID=1327a02b:b19f6696:0e3f8ac7:9615591c

Growing RAID size

This is useful if the RAID array needs to be grown by using up more free space (no added harddrive)

umount /dev/sda
umount /dev/sdb
umount /dev/sdc
umount /dev/sdd

# grow RAID array to 500GB (this will take a while)
mdadm -G /dev/md127 -z 500G

# resize physical volume to fit new RAID partition size
pvresize /dev/md127

Accessing via Live CD

If the array gets screwed up somehow, you can try mounting it on a livecd.

apt install mdadm

# assemble array from block devices
mdadm --assemble --scan

# mount array (assuming lvm)
apt install lvm2

# see if lv's are intact

# mount lv
mount /dev/[vgname]/[lvname] /mnt/foo

Installing GRUB on a Live CD Mounted System

# mount root lv
mount /dev/[vgname]/root /mnt/root

# mount live CD directories inside mounted lv
for i in /dev /dev/pts /proc /sys /run; do sudo mount -B $i /mnt/root$i; done

# chroot into root lv
chroot /mnt/root

# install grub to each device in array
grub2-install /dev/sda
grub2-install /dev/sdb
grub2-install /dev/sdc
grub2-install /dev/sdd

# update grub config
grub2-mkconfig -o /boot/grub2/grub.cfg

Mounting Images

# list the partitions on the image file
fdisk -l /tmp/sdcard.img 
Disk /tmp/sdcard.img: 162 MiB, 169869824 bytes, 331777 sectors
Units: sectors of 1 * 512 = 512 bytes
Sector size (logical/physical): 512 bytes / 512 bytes
I/O size (minimum/optimal): 512 bytes / 512 bytes
Disklabel type: dos
Disk identifier: 0x00000000

Device           Boot Start    End Sectors  Size Id Type
/tmp/sdcard.img1 *        1  65536   65536   32M  c W95 FAT32 (LBA)
/tmp/sdcard.img2      65537 331776  266240  130M 83 Linux
# use the sector size and the partition start sector to calculate offset (512 * 65537)
sudo mount -o loop,offset=33554944 /tmp/sdcard.img /mnt/tmp

Auto FS

Auto FS + SSHFS allows the system to mount ssh filesystems on access and then automatically unmount after a certain timeout. The necessary tools are autofs and sshfs.

/etc/auto.master or /etc/auto.master.d/foobar.autofs or /etc/autofs/auto.master

# mounts all the entries listed in /etc/auto.sshfs in /mnt/ with the given options
# add the --verbose option here to debug mounting issues
# set --timeout to control when sshfs mount is automatically unmounted
/mnt /etc/auto.sshfs --timeout=180 --ghost


# make a mount to be used by auto.master
foobar -fstype=fuse,rw,IdentityFile=/home/evan/.ssh/foobar,port=22,allow_other :sshfs\\:

AutoFS runs as root, so ensure that the host fingerprint has been added to /root/.ssh/knownhosts. You can add this easily by attempting to ssh login to from root.

su -
# enter yes

Resizing LUKS encrypted LVM

# expand the block device with fdisk, if necessary

# resize physical volume
pvresize --setphysicalvolumesize 111.8G /dev/sdb2
# be careful about using `-l +100%FREE`.  this broke /home until I manually shrank fedora--vg-home by a few GB
lvextend -l 80G /dev/mapper/fedora--vg-home
resize2fs /dev/mapper/fedora--vg-home

Fixing Nodejs


# Sync permissions only. (useful if you forgot `-p` option in cp)
# Looks at filesize differences to determine if a copy is needed rather
# than timestamp (which gets reset when `-p` is left out of cp.
rsync --archive --size-only /src/foo /dest/bar

DNS Tunneling with iodine

Most of this was taken from

Domain Setup

On a domain you own (e.g., create an A record pointing to the ip of a server you own and an NS record pointing to

To verify the setup is working, you can do:

# on the server
sudo nc -u -l -p 53

# on another device
dig +trace
# you should see some stuff printed out in the console on the server

Server Setup

# install iodine
dnf install iodine

# run iodine (as root in a screen session)
#  `password` is the password to use the tunnel
#  `` is the ip of the server on the tunnel network
iodined -c -P password -f

# set iptables rules
iptables -t nat -A POSTROUTING -o eth0 -j MASQUERADE
iptables -A FORWARD -i eth0 -o dns0 -m state --state RELATED,ESTABLISHED -j ACCEPT
iptables -A FORWARD -i dns0 -o eth0 -j ACCEPT

# enable ip forwarding on the server
#  unnecessary if you want to use `ssh -D 1234` for dynamic port forwarding 
#  on the client (as opposed to setting default routes)
echo 1 > /proc/sys/net/ipv4/ip_forward

Client Setup

Alternatively, you can download a script that does this part from

# launch iodine client and wait for a 'Connection setup complete' message
sudo iodine -f

# either use SSH for dynamic forwarding (one application at a time)  or set up routes

# ssh
ssh -D 1234
# set Firefox to use socks proxy localhost on port 1234

# set up routes
# get the current gateway ip
ip route
# get the tunnel server ip
# add a route for iodine to communicate with the outside world
sudo ip route add [ IP] via [current gateway IP]
# delete the default route for traffic
sudo ip route delete default
# add a default route so that all traffic is tunneled
sudo ip route add default via


block - smallest addressable unit of storage

Block size is defined in the hardware of a hard drive, but the OS can define a virtual block size which chains multiple blocks together.

There are three primary boot options involving UEFI and BIOS firmwares

    • required if dualbooting windows
    • max addressable disk space is 232 * 512 = 2 TiB on a system with 512 byte blocks.

GPT - GUID Partition Table

protective mbr - a small partition at the beginning of the GPT disk (where the MBR would normally be) that prevents older MBR tools from damaging the GPT formatting

This partition contains a fake partition record which spans the entirety of the disk. MBR programs will see that there is a partition of unknown type that spans the entire disk and will refuse to operate.

A GPT disk is formatted like so:

Protective MBR 512B
GPT Header 512B
GPT Partition Table 16KB
* Partitions *
Backup Partition Table 16KB
Backup Header 512B

So there should be 17KB and 16.5KB of free space at the beginning and end of a GPT disk.

Random facts

  • grub2-install invokes efibootmgr to install (aka register) entries in the nvram
  • these nvram entries point to .efi executables on the ESP
  • the harddrive UEFI menu entries are for legacy booting these devices
  • efi/boot/bootx64.efi is the .efi executable location for removable devices and doesn't require any nvram registration

SMART Status

smartctl -a /dev/sdX

smartctl -t short /dev/sdX

Network interfaces and bridging

Simulating network disconnect

# add network namespaces (for network isolation)
sudo ip netns add client-ns
sudo ip netns add server-ns

# create pairs of virtual interfaces
sudo ip link add client type veth peer name client-bridge
sudo ip link add server type veth peer name server-bridge
# add virtual interfaces to namespace
sudo ip link set client netns client-ns
sudo ip link set server netns server-ns
# give addresses to each virtual interface
sudo ip netns exec client-ns ip addr add dev client
sudo ip netns exec server-ns ip addr add dev server
# set virtual interfaces up
sudo ip netns exec client-ns ip link set client up
sudo ip netns exec server-ns ip link set server up
sudo ip link set client-bridge up
sudo ip link set server-bridge up

# add bridge interface
sudo brctl addbr bridge0
# link virtual interfaces to bridge
sudo brctl add if bridge0 client-bridge
sudo brctl add if bridge0 server-bridge
# set bridge up
sudo ip link set bridge0 up

# (in a new terminal) do stuff on the virtual interface
sudo ip netns exec client-ns ping

# set bridge down (simulate network offline)
sudo ip link set bridge0 down

# sometimes you might need to use the loopback interface
sudo ip netns exec client-ns ip link set lo up
sudo ip netns exec server-ns ip link set lo up