Linux command bible

Map alike

Example:

declare -A PORT_TO_APPS=(
    ["8090"]="app1"
    ["8182"]="app2"
 )
for port in "${!PORT_TO_APPS[@]}";
do
    echo "========${port} : ${PORT_TO_APPS[${port}]}===================="
    sudo fuser ${OPTS} ${port}/tcp
    echo " "
done

Install a .deb file along with all of its (missing) dependencies

Use gdebi instead of dpkg

Sudo GUI

gksu

Search for filename

locate mysql | grep ^.*/mysql$

Search for texts in files in a folder recursively

grep -I -R "text to search for" /a/b/c
grep -I -R "port=5001" --include=*.py ./

Create fake package (with only dependencies)

mkdir package-name
cd package-name
equivs-control controlfile

...edit that control file, fill information: equivis-build controlfile

Untar

  • Extract all to current dir: tar -zxvf file.tar.gz
  • Extract 1 file: tar -zxvf file.tar.gz file-can-extract
  • Extract 1 group of file boi pattern: tar -jxvf file.tar.bz2 --wildcards "s*"

tar

  • tar+gzip
    • Being in the folder containing the file need to tar: tar czf output.tar.gz input-file-or-folder
    • From a folder, tar another folder: tar -C /path/to/folder/containing/file/or/folder/to/tar -czf /output/folder/output.tar.gz filename-or-folder-name-to-tar
  • list content: tar -tvf file-tar
  • Update (if exists) or append new file to tar file: tar -rvf file.tar file-can-update
  • Tar - split - merege
    • split: tar -cvpz /path to zip | split -d -b 3900m - /name/of/backup.tar.gz
    • merge and untar: cat *tar.gz* | tar -xvpzf - -C /

CPU

  • OS info 32/64 bit: uname -a
  • CPU info: lscpu

View memory usage

  • free 
    free -m
# every 5 seconds and output to console
watch -n 55 free -m run "free -m"
  • Actual used = Used - (buffer + cached)
  • Actual free = Free + (buffer + cached)

  • ==> just look at line: -/+ buffer/cache

  • top then press m
  • ps 
    ps aux | awk '{print $4"\t"$11}' | sort | uniq -c | awk '{print $2" "$1" "$3}' | sort -nr
# or
ps -eo pmem,pcpu,size,pid,cmd | sort -k 1 -nr | head -5
# (size in KB)

chmod only directory/file

  • Directory only: chmod 755 $(find . -type d)
  • File only: chmod 755 $(find . ! -type d)

Ports

  • List open ports

    fuser -v 8080/tcp
# kill
fuser -vk 8080/tcp
# or
netstat -lnptu | grep 8080
# or
lsof -i :8080
  • All open ports: netstat -ntlp | grep LISTEN
  • Port scan: nc -zv <IP> <port range>

Date

  • view: date
  • change: sudo date MMddhhmmYYYY.ss
  • timezone: sudo dpkg-reconfigure tzdata

Disk

  • Disk usage: df -h
  • Monitor IOs: sudo iotop
  • Folder usage: du -m --max-depth=1 /home | sort -nk1

Mount

  • Mount to samba:
# create mount point (for example)
sudo mkdir /mnt/samba
sudo chattr +i /mnt/samba

sudo mount.cifs //<IP>/sharedfolder /mnt/samba -o user=..,password=..,uid=..
  • Mount NFSv4

see http://manpages.ubuntu.com/manpages/bionic/man5/systemd.mount.5.html

or: man systemd.mount

sudo apt install nfs-common
# list all exported NFS directories on the NAS server 
showmount --exports NFS-server-IP

# create mount point
sudo mkdir /mnt/nfs
sudo chattr +i /mnt/nfs

# /etc/fstab
NFS-server-IP:/path/the/share /mnt/nfs nfs nfsvers=4,clientaddr=IP-of-this-manchine,rw,suid,dev,user,exec,sync,_netdev,noatime,proto=tcp,sec=sys,x-systemd.after=network-online.target,x-systemd.device-timeout=5s,x-systemd.automount      0 0

systemctl daemon-reload

Text search and replace

  • replace "ugly" with "beautiful": sed -i 's/ugly/beautiful/g' <text file>
  • Search string in file, print out line: grep -in "string" <file>
  • Show line in text file: sed -n '10000000,10000020p' filename
  • Delete the first line: sed -i '1d' filename
  • Convert DOS to unix: tr -d '\015' < dos-file > output-unix-file
  • Insert "text" to the beginning of a file. sed -i '1s/^/text /' file
  • Append "text" to the end of a file: echo "text" >> file

Symbolic link

  • create a "folder" inside "container": n -s source/folder dest/container
  • link "here" to "folder": ln -sf source/folder here
Symbolic link Hard link
  • Points to another file by name
    		•
  • Points to another file by inode number original file == link, no difference
    		•
  • Linked file: change content without affecting the symbolic link
    		•
  • Change file name, still points to that file
    		•
  • Linked file doesn't have to exist
    		•
  • Must exist, on same filesystem
    		•

    Stream & redirection

    • 0 stdin file descriptor
    • 1 stdout file descriptor
    • 2 stderr file descriptor

    • Send output of a background process to file: some_command > some_file.txt 2>&1 & Explain:

    • First, redirect stdout to the file
    • Then redirect stderr to stdout, it, in turn, will be forwarded to the file
    • Why 2>&1 but not 2>1: the & is to indicate that is a file descriptor. Without that, it would be: redirect the stderr to a file named 1.
    • Last, the & send the projects to background

    Prompt for user input

    OPTIONS=("Yes - ${STUDYMAP_WEB_VERSION} is the correct version" "No - wrong version")

select CHOICE in "${OPTIONS[@]}"
do
  echo "Your chosen answer: $CHOICE"
  case $CHOICE in
        "Yes - ${STUDYMAP_WEB_VERSION} is the correct version")
            build
            break
            ;;
        "No - wrong version")
            exit 0
            break
            ;;
        *)
            echo "Wrong option"
            exit 1
  esac
done

    Select a file from directory

    # List all config properties files
# causes the array to be empty if there are no files found with globbing
pushd .
cd ${DB_COPY_HOME}
shopt -s nullglob
CONFIG_FILES=( config*.properties )
echo "files: ${CONFIG_FILES[@]}"
popd

# Choose one of the config files
echo 'Select the config file you want to run with'
select opt in "${CONFIG_FILES[@]}" "Quit"; do
  case ${opt} in
    *.properties)
        break
        ;;
    "Quit")
        echo 'Stop the script without copying anything'
        exit 0
        ;;
    *)
        echo "Wrong choice. Please choose one of numbers listed above"
        ;;
  esac
done

    Parse command line arguments

    Use builtin bash function getopt which is standard, not getopts bash function

    function usage() {
    echo "Usage: $0
        -k <ssh private key> optional, only require if the remote host requires
        -h <remote host> required
        -u <remote username> required
        " 1>&2;
    exit 1;
}
# the leading ":": switch off error reporting mode of getopt since we will handle it by ourselves
# ":" after an option: that option expect an argument, which will be passed to OPTARG variable
while getopts ":h:u:" option; do
    case ${option} in
        k)
            SSH_KEY=${OPTARG}
            ;;
        h)
            REMOTE_USER=${OPTARG}
            ;;
        u)
            REMOTE_HOST=${OPTARG}
            ;;
        # This to catch invalid options
        \?)
            # in this case, OPTARG contains the option name
            echo "Invalid option: -$OPTARG" >&2
            usage
            ;;
        # this to catch valid option however an argument is required but missing
        :)
            echo "Option -$OPTARG requires an argument." >&2
            usage
            ;;
    esac
done
if [ "${REMOTE_USER}" == "" ]; then echo "Remote username is expected"; usage; fi
if [ "${REMOTE_HOST}" == "" ]; then echo "Remote host is expected"; usage; fi

    Networking

    • List all available network cards (both configured or not configured): ifconfig -a
    • Capture traffic to a host via an network interface sudo tcpdump -i <interface name> -n host <host IP>
    • Show all network bridges brctl show docker0. More about networking and bridges

    USB Modem 3G/4G

    For example Huawei USB stick modem

    See Device Reference of usb_modeswitch

    sudo apt-get install usb-modeswitch

# plug the usb stick in.
# It should be switch automatically to Modem/Networkcard mode from (mass storage mode)

# Otherwise,
# List all USB devices. Find out the Huawei line and take note the <vendor ID>:<product ID>
lsub

# For example
# Bus 001 Device 003: ID 12d1:14db Huawei Technologies Co., Ltd. E353/E3131
#
# 12d1: vendor ID of Huawei
# 14db: is the product ID when it was switched to Modem/Networkcard mode
#       if it is still in mass storage mode, its ID will be: 1f01

# To switch from Mass-storage mode to Modem/Networkcard mode
sudo usb_modeswitch -default-vendor 12d1 --default-product 1f01 --huawei-new-mode

# Determine the ethernet card that the modem is registering
dmesg
#
# catch the line like
# [ 4988.931717] cdc_ether 3-6:1.0 <ethernet card name is here>: register 'cdc_ether' at usb-0000:00:14.0-6, CDC Ethernet Device, **:**:**:**:**:**

# register the card name in netplan yml file. Something like
network:
    version: 2
    renderer: networkd
    ethernets:
        enp3s0:
            dhcp4: true

# Then
sudo netplan apply

    SSH

    Generate ssh keys and configure authentication agent

    ssh-keygen -t rsa -f ~/.ssh/tribao-bit -C "[email protected]" 
chmod 600 ~/.ssh/tribao-bit
# start the agent AND set environment variables
eval `ssh-agent -s`
# add ssh key to the agent
ssh-add ~/.ssh/tribao-bit

    To start ssh-agent automatically:

    • Edit file ~/.profile

      if [ -z "$SSH_AUTH_SOCK" ] ; then
eval `ssh-agent -s`
ssh-add
fi
    • Edit file ~/.ssh/config 
      Host bitbucket.org
HostName bitbucket.org
IdentityFile ~/.ssh/tribao-bit
User tribao

    VIM commands

    :set number ==> show line number
    :<number> ==> move to line <number>
    dd ==> delete the current line
    d$ ==> delete to the line end
    d^ ==> delete to the line start
    $ ==> go to end line
    :$ ==> go to end file
    0 ==> go to line start
    :0 ==> go to start file
    i ==> switch insert mode
    esc ==> escape insert mode
    /phase ==> search for phase
    u ==> undo

    List file by size (used to find large files)

    # list all file >5MB from the current directory
find ./ -type f -size +5000k -exec ls -lh {} \; | awk '{ print $9 ": " $5 }'

    Download file

    curl -o <file to write to local disk> -SL <url>

wget -O <file to write to local disk> -S <url>

    ulimit

    • Number of open files: defined by linux kernel an couldn't be set to -1 or unlimited like other limits. The max value can be seen by 
      $ cat /proc/sys/fs/nr_open
1048576

    User

    Add a login user

    # with sudo permission
sudo useradd -G sudo -m -s /bin/bash <username>
sudo passwd <username>

    other

    https://stackoverflow.com/questions/59895/get-the-source-directory-of-a-bash-script-from-within-the-script-itself

    Add disk

    # determine device name (e.g. /dev/sdb)
sudo lsblk --fs

# optional, used when the disk has already had filesystem
sudo wipefs <device name>

#make partition
# the following sfdisk commands won't work if the disk is > 2TB. use parted
#sudo sfdisk <device name> << EOF
#;
#EOF
sudo parted <device name>
    (parted) print
    (parted) mklabel gpt
    (parted) mkpart
        Partition name?  []?
        File system type?  [ext2]? ext4
        Start? 0%
        End? 100%
    (parted) print

# For disks that have already parted, use fdisk, as following
sudo fdisk <device name>
# then following the interaction instruction

# create ext4 filesystem
sudo mkfs.ext4  <device name>1

# reclaim all reserved blocks - 5%
sudo tune2fs -m 1 <device name>1

#create mount point
sudo mkdir /<mountpoint dir>
# this command will make the mount point director immutable for all user even root!
# only mutable when a filesystem is mounted on it. Use it instead of "sudo mkdir -m 000 /<mountpoint dir>"
sudo chattr +i /<mountpoint dir>

# Get UUID of the new partition
lsblk -f

# /etc/fstab
# use UUID instead of device name because if using something like /dev/sdb1,
# when a new disk is installed on the server or the existing disk was relocated
# (swap cable), the device name will changed!!!
UUID=<fill the UUID> /<mout point> auto defaults 0 2

sudo mount <mout point>

    Systemd useful commands

    # system boot-up performance statistics
sudo systemd-analyze blame

# view log of one service, tail and follow
sudo journal -f -u docker.service

# show log of one service, newest lines first
sudo journalctl -r -u docker.service

# show kernel logs from previous boot
# Augment log lines with explanation texts from the message catalog (-x)
sudo journalctl -x -k -b -1

# view disk consumption. It shows the sum of the disk usage of both ARCHIVED and ACTIVE journal files,
sudo journalctl --disk-usage

# all currently active journal files are marked as archived and renamed (--rotate)
# remove archived journal files until the disk space they use is under 500M
sudo journalctl --rotate --vacuum-size=500M