Practical Commands
This guide has all the *nix commands you can practically use.
By Omkaram - Dec 14, 2024
Rsync
Useful to copy or transfer files locally or via a network. Add --dry-run for dry run
# Display progress while transferring and exclude list of file or folders from transferring
$ rsync -rahuv --info=progress2 --stats --exclude-from=exclude-files-folders-list.txt [source] [target]
SCP
Secure Copy. Helps to transfer files especially over a SSH tunnel network.
# Recursively copy files from Remote to local
$ scp -i "yourkey.pem" -r [username]@[IP]:~/[folder on the server]/* [your local destination]
# Recursively copy files from local to Remote
$ scp -r [your local destination] [username]@[IP]:~/[folder on the server]/*
Process Monitoring
Secure Copy. Helps to transfer files especially over a SSH tunnel network.
# Prints the processes Start time, PID, CPU%, MEMORY%, MEMORY, COMMAND
$ ps aux | gawk '{mem[$11]+=int($6/1024)}; {time[$11]=$9; cpuper[$11]+=int($3)}; {memper[$11]+=int($4)}; {id[$11]=$2}; END {for (i in id) {print strftime("%Y-%m-%d %T") "\t" time[i] "\t" id[i] "\t" cpuper[i]"% \t",memper[i]"%\t",mem[i]" MB\t",i,"\t\t"}}'
====
# results
2024-12-13 21:14:39 18:54 1994 0% 0% 20 MB /usr/libexec/xdg-desktop-portal-gtk
2024-12-13 21:14:39 18:52 659 0% 0% 0 MB [zvol]
2024-12-13 21:14:39 18:51 6 0% 0% 0 MB [netns]
2024-12-13 21:14:39 18:52 1055 7% 0% 113 MB /usr/lib/xorg/Xorg
2024-12-13 21:14:39 18:51 112 0% 0% 0 MB [vfio-irqfd-clea]
2024-12-13 21:14:39 18:54 1788 0% 0% 23 MB /usr/bin/xembedsniproxy
2024-12-13 21:14:39 18:51 136 0% 0% 0 MB [kworker/u33:0-hci0]
2024-12-13 21:14:39 18:51 25 0% 0% 0 MB [cpuhp/2]
2024-12-13 21:14:39 18:51 22 0% 0% 0 MB [ksoftirqd/1]
===
# You can use the same in a loop every 10 mins to monitor your processes
in my example "openjdk" in a detaches state and write it to a file
# create a file name "process-mon.sh" and place this below in it and run nohup ./process-mon.sh &
while true;
do
sleep 600
ps aux | gawk '{mem[$11]+=int($6/1024)}; {time[$11]=$9; cpuper[$11]+=int($3)}; \
{memper[$11]+=int($4)}; {id[$11]=$2}; END {for (i in id) {print strftime("%Y-%m-%d %T") "\t" time[i] "\t" id[i] "\t" cpuper[i]"% \t",memper[i]"%\t",mem[i]" MB\t",i,"\t\t"}}' \
sort -k3nr | grep "openjdk" | tee -a "mule-memory.log"
done
Find
This command is useful to find files and can be used in conjunction with AWK and SED
# Looks for the word linuxmule.com in all the files of current directory
# and print the file name, number and match
$ find ./ -type f -exec awk '/linuxmule.com/ { printf "%s *** %s *** %s\n\n",FILENAME,FNR,$0 }' '{}' +
====
# Result of the above
/articles/index.php *** 24 *** link rel=alternate hreflang=en href="https://linuxmule.com/" /
./index.php *** 8 *** link rel=alternate hreflang=x-default href="https://linuxmule.com/" /
./index.php *** 9 *** link rel=alternate hreflang=en href="https://linuxmule.com/" /
./work/index.php *** 8 *** link rel=alternate hreflang=x-default href="https://linuxmule.com/" /
./work/index.php *** 9 *** link rel=alternate hreflang=en href="https://linuxmule.com/" /
# If you want to work on multiple files in the same folder and subsequent folders,
# then either one or both of the two commands given below must work
$ sed -i 's/abc/omkaram/g' *
$ find ./ -type f -exec sed -i 's/Hello world/Omkaram Venkatesh/g' {} \;
====
# But what if you do not want to replace a string with another string, instead just what to find the matching words in a file? Like our AWK example?$_COOKIE
$ find ./ -type f -exec sed -n 's/.*\(omkaram\).*/\1/p' {} \;
lsof
LSOF list open files. Helpful to find open files.
# To check total files opened by top 4 processes
$ lsof | awk '{printf "Files open | Program: %s | PID: %s\n", $1, $2}' | sort | uniq -c | sort -rn | head -n 4 | column -t
====
# Result of the above
68706 Files open | Program: firefox-b | PID: 4889
31174 Files open | Program: plasmashe | PID: 1785
14760 Files open | Program: code | PID: 3613
10051 Files open | Program: krunner | PID: 2693
# To check files opened by a user OR specific directory
$ lsof -u [USER_NAME] +D [location]
---
# To check files opened by a user AND IN a specific directory
$ lsof -u [USER_NAME] -a +D [location]
---
# To check files not open by a user AND IN a specific directory
$ lsof -u ^[USER_NAME] -a +D [location]
---
# To check all files opened by a user and TCP and UDP connections
$ lsof -u [USER_NAME] -i TCP -i UDP
---
# To check all files NOT opened by a user and TCP and UDP connections
$ lsof -u ^[USER_NAME] -i TCP -i UDP
---
# To check all files NOT opened by a user and TCP and UDP connections, without hostnames(-n) (i for ipv[46] files)
$ lsof -u ^[USER_NAME] -i TCP -i UDP -n
---
# To check all files NOT opened by a user and TCP connections, without hostnames and no portnames (want port numbers with -P)
# and then filter for TCP Listening connections
$ lsof -u ^[USER_NAME] -i TCP -nP -s TCP:LISTEN
---
# To check all LISTEN and ESTABLISHED conns on port range
$ lsof -n -i:80-4431
---
# To check all LISTEN and ESTABLISHED conns by a PID (-a means AND)
$ lsof -Pan -p 1077 -i
---
# To check all LISTEN and ESTABLISHED conns by a USER
$ lsof -u mysql -Pan -i
Page 1
of 2