Monday 30 March 2020

Bash script to remove duplicate files

Identify the duplicate files and remove.

vi remove_duplicate.sh
#!/bin/bash
ls -ls --time-style=long-iso | awk 'BEGIN {
 getline; getline;
 name1=$9; size=$5
}
{
 name2=$9;
 if (size==$5)
 {
  "md5sum "name1 | getline; csum1=$1;
  "md5sum "name2 | getline; csum2=$1;
  if ( csum1==csum2 )
        {
         print name1; print name2;
        }
 };

 size=$5; name1=name2;
}' | sort -u > duplicate_files

cat duplicate_files | xargs -I {} md5sum {} | sort | uniq -w 32 | awk '{ print "^"$2"$" }' | sort -u > duplicate_sample

if [ -s duplicate_sample ]
 then
  echo Removing...
  comm duplicate_files duplicate_sample -2 -3 | tee /dev/stderr | xargs rm -rf
  echo Removed duplicate files successfully.
 else
 echo No duplicate files found.
fi

Friday 27 March 2020

20 Examples of Linux xargs Command


 xargs Command usage

man xargs
xargs --version

ls -l /usr/local/src/ | xargs                #single line output
cat example.txt | xargs -n 7
xargs -a example.txt
echo 'one two three' | xargs mkdir #creating three dirs
echo 'one two three' | xargs -p touch #creating three files

printf "one  three  two" | xargs -i touch {}.txt #add extention
echo "Text1XText2XText3XText4" | xargs -d X -n 2
printf %s\\n {1..30} | xargs -n 5 -P 8                     #executing commands in parallel
cat args.txt | xargs -n 2 '/usr/local/src/cecho.sh'

cat args.txt | xargs -I {} ./cecho.sh -p {} -l
echo "file1 file2 file3" | xargs -t -I % sh -c '{ touch %; ls -l %; }' #multiple commands
find ./*.txt -print0 | xargs -0 -n 1 -P 3 bzip2    #bzip2 processes in parallel
ls "b.txt" | xargs -n 1 sed -i "s/color/colour/g" #replace all occurrences

cut -d: -f1 < /etc/passwd | sort | xargs #compact list of all Linux users
ls *digital* | xargs wc                             #number of lines/words/characters in each file
echo dir1/ dir2/ dir3/ | xargs -n 1 cp -v dir/test.txt #copy a file to multiple dirs at once
find . -type f -name “*.java” | xargs tar cvf myfile.tar
find / -type f -name "*.sh" -print0 | xargs -0 wc -l #cout no of lines

find . -type f -name "*.txt" -print0 | xargs -0 rm -f                 #remove files
find . -type f -name "b.txt" -print0 | xargs -0 -p -n 1  rm -rf   #prompt before execution
find /tmp -mtime +7 | xargs rm

find . -type f -not -name '*.txt' -print0 | xargs -0 -I {} rm -v {} #remove except extension
find . -type d -name "abb" -print0 | xargs -0 rm -v -rf "{}"        #remove directory
echo 'one two three' | xargs -t rm -rf                               #remove three files/dirs

Ref:- linuxtechi.com shapeshed.com tecmint.com


Thursday 26 March 2020

60 Examples of Linux find command

Using find command in Linux

Find becomes extremely useful when combined with other commands,

man find
info find
find . -type d -print  #list directories
find . -type f -print  #regular files
find . -type l -print  #symbolic files

find . -name demo.sh
find . -iname demo.sh #case-insensitive search
find /home -iname demo.sh      #both capital and small letters
find . -exec ls -ld {} \;         #List out the found files
find . \( -name "*.mp4" -o -name "*.txt" \) -print
find /home/test -path "*/public/*" -print
find . -iregex ".*\(\.py\|\.sh\)$"
find . -type f -not -name "*.sh" #invert match
find . ! -name "*.txt" -print

find /tmp -type f -empty         #find Empty files
find /tmp -type d -empty find   #Empty directories
find /tmp -type f -name ".*"  #find hidden files
find ./ -type f -name "ab.txt" -exec grep 'This' {} \; #print lines which have ‘This’
find ./test -name 'abc*' ! -name '*.php' #combine multiple search criteria
find -name '*.php' -o -name '*.txt'
find ./test ./dir2 -type f -name "abc*"       #search multiple directories
find . -maxdepth 1 -name "f*" -print
find . -mindepth 2 -name "f*" -print

find . -type f -atime -7 -print    #accessed in last 7 days
find . -type f -atime 7 -print     #accessed in last 7th day
find . -type f -atime +7 -print    #accessed older that 7 days
find / -mtime +7 –mtime -10     #modified 7 days back and less than 10 days
find / -cmin -60  #files which are changed in last 1 hour
find / -mmin -60 #the files which are modified in last 1 hour
find / -amin -60 #files which are accessed in last 1 hour
find . -type f -amin +7 -print                  #accessed in last 7 mins
find . -type f -newer cal_seq.sh -print    #newer than a file

find . -type f -size +2k                      #file size
find / -size +50M -size -100M             #greater than 50MB and less than 100MB
find . -type f -exec ls -s {} \; | sort -n -r | head -5  #find largest
find . -type f -exec ls -s {} \; | sort -n | head -5     #smallest file
find . -type f -name "a.txt" -exec cp {} test/ \;     #find and move
find / -type f -size +100M -exec rm -f {} \;               #find all 100MB files and delete
find / -type f -name *.mp3 -size +10M -exec rm {} \;#find all .mp3 & delete more than 10MB
find / -name name.txt -exec rm -i {} \; #delete a file with confirmation
find . -type f -name *.swp -delete                 #delete

find / -perm 1551 #sticky bit files
find / -perm /u=s #SUID
find / -perm /g=s #SGID
find / -perm /u=r #read only
find / -perm /a=x # executable

find /etc -maxdepth 1 -perm /u=r
find . -type f -perm 644 -print    #permission
find / -type f ! -perm 777 #with out permission
find . -type f -name "*.php" ! -perm 644 -print #don't have permission levels
find / -type f -perm 0777 -print -exec chmod 644 {} \;  #change permission
find . -type f -user root -print #find based on user
find /data -owner smith #find all files owned by smith
find /home -group developer #find based on group
find . -type f -user root -exec chown test {} \; #find and change ownership

find ./ -type f -exec sed -i 's/find/replace/g' {} \; #find and replace
find / -name \*.txt -exec sed -i "s/one/five/g" {} \;
find . -type f -name "*.c" -exec cat {} \;>all_c_files.txt #all .c to .txt
find . -type f -mtime +10 -name "*.txt" -exec cp {} OLD \; #cp 10 days old OLD dir
find . -type f -name "*.txt" -exec printf "Text file: %s\n" {} \;  #list .txt

find / -type f -name "*.sh" -print0 | xargs -0 wc -l           #cout no of lines
find . -type f -name "*.txt" -print0 | xargs -0 rm -f          # remove files
find . -type f -name “*.java” | xargs tar cvf myfile.tar    #find and tar files
find [paths] [expression] [actions] 2>/dev/null          #finding error messages
find ~/documents -type f -name '*.txt' \ -exec grep -s DOGS {} \; -print   #grep

#remove duplicate files
find . ! -empty -type f -exec md5sum {} + | sort | uniq -w32 -dD
find -not -empty -type f -printf "%s\n" | sort -rn | uniq -d | xargs -I{} -n1 find -type f -size {}c -print0 | xargs -0 md5sum | sort | uniq -w32 --all-repeated=separate

Monday 23 March 2020

tput in Linux

tput usage

$ man tput
$ info tput

$ tput longname #full name of the current terminal type
xterm terminal emulator (X Window System)

$ tput -T screen longname
VT 100/ANSI X3.64 virtual terminal

Foreground & background color commands

tput setab [1-7] # Set the background color using ANSI escape
tput setaf [1-7] # Set the foreground color using ANSI escape

Colors are as follows:

No   Color        #define                R G B

 0    black       COLOR_BLACK        0,0,0
 1    red           COLOR_RED          1,0,0
 2    green       COLOR_GREEN       0,1,0
 3    yellow      COLOR_YELLOW     1,1,0
 4    blue         COLOR_BLUE          0,0,1
 5    magenta   COLOR_MAGENTA   1,0,1
 6    cyan        COLOR_CYAN           0,1,1
 7    white       COLOR_WHITE         1,1,1

















Text mode commands

tput bold    # Select bold mode
tput dim     # Select dim (half-bright) mode
tput smul    # Enable underline mode
tput rmul    # Disable underline mode
tput rev     # Turn on reverse video mode
tput smso    # Enter standout (bold) mode
tput rmso    # Exit standout mode

Cursor movement commands

tput cup Y X # Move cursor to screen postion X,Y (top left is 0,0)
tput cuf N   # Move N characters forward (right)
tput cub N   # Move N characters back (left)
tput cuu N   # Move N lines up
tput ll      # Move to last line, first column (if no cup)
tput sc      # Save the cursor position
tput rc      # Restore the cursor position
tput lines   # Output the number of lines of the terminal
tput cols    # Output the number of columns of the terminal

Clear and insert commands

tput ech N   # Erase N characters
tput clear   # Clear screen and move the cursor to 0,0
tput el 1    # Clear to beginning of line
tput el      # Clear to end of line
tput ed      # Clear to end of screen
tput ich N   # Insert N characters (moves rest of line forward!)
tput il N    # Insert N lines

Reset and play bell commands

tput sgr0    # Reset text format to the terminal's default
tput bel     # Play a bell

Error Messages

tput displays various error messages if problems occur. In addition, it exits with one of the following status values:

0 Normal status; the given capability is present.
1 The given Boolean or string capability is not present.
2 Usage error; tput was given invalid arguments.
3 The terminal type given is unknown, or the termcap database can not be read.
4 The given capability is unknown.


Ref:- stackoverflow.com gnu.org linuxcommand.org

Linux date command usage

Date command usages in Linux

The date command displays or sets the system date and time. It is most commonly used to print the date and time in different formats and calculate future and past dates.

$ man date
$ info date

$ date
Mon Mar 23 08:08:21 EDT 2020

$ date +"Year: %Y, Month: %m, Day: %d"
Year: 2020, Month: 03, Day: 23

$ date "+DATE: %D%nTIME: %T"
DATE: 03/23/20
TIME: 08:08:41

$ date -d "2010-02-07 12:10:53" #date string option
Sun Feb  7 12:10:53 EST 2010

$ date -d '16 Dec 1974' +'%A, %d %B %Y'
Monday, 16 December 1974

$ date -d "last week"             #Display past date
Mon Mar 16 08:10:54 EDT 2020

$ date -d 'TZ="Australia/Sydney" 06:30 next Monday'
Sun Mar 29 15:30:00 EDT 2020

$ TZ=GMT date
Mon Mar 23 12:15:31 GMT 2020

$ TZ='Australia/Melbourne' date #current time of some other location
Mon Mar 23 23:11:29 AEDT 2020 (ls /usr/share/zoneinfo)

$ date -u               #Display universal time
Mon Mar 23 16:57:45 UTC 2020

$ date --iso-8601=seconds #ISO 8601 format
2020-03-23T13:06:30-0400

$ date --rfc-3339=seconds #RFC 3339 format
2020-03-23 13:08:54-04:00

$ date +%s                #Epoch converter (seconds since 00:00:00, Jan 1, 1970)
1584965527

$ date -d @1234567890         #Convert epoch to a date
Fri Feb 13 18:31:30 EST 2009

$ date -d "2020-01-01" +"%s" #calculate the seconds from epoch to provided date/time
1577854800

$ date_now=$(date "+%F-%H-%M-%S") #use with shell scripts
$ echo $date_now
2020-03-23-08-13-50

$ date -r /etc/hosts                 #Last Modification Time of a File
Wed Jan 23 17:30:26 EST 2019

$ date --set="20200801 12:30" #Set the System Time and Date

$ date +"Week number: %V Year: %y"
Week number: 13 Year: 20

$ touch ~/Desktop/`date +%F`.txt #create a file with the date and time

$ tar cfz /backup-`date +%F`.tar.gz /home/ #create archives with the date and time

$ mysqldump  db_name > db_name-$(date +%Y%m%d).sql #mysqldump file with date


Sunday 22 March 2020

Color code in Linux shell

Color modes, colors for bash prompt, tput and ANSI color 

Setting colors in terminal session is only temporary and relatively unconditional.
Modern systems usually default to at least xterm-256color.

$ echo $TERM
xterm (old)

Check current settings

$ echo $LS_COLORS
rs=0:di=01;34:ln=01;36:mh=00:pi=40;
33:so=01;35:do=01;35:bd=40;33;01:cd=40;
[...]

$ dircolors --print-database
[...].jpg 01;35
.jpeg 01;35
.gif 01;35[...]

$ dircolors --bourne-shell
LS_COLORS='rs=0:di=01;34:ln=01;36:mh=00:pi=40;
33:so=01;35:do=01;35:bd=40;33;01:cd=40;
[...]

The Color modes:

1. Color-mode

It modifies the style of color NOT text. For example make the color bright or darker.

0; reset
1; lighter than normal
2; darker than normal
This mode is not supported widely. It is fully support on Gnome-Terminal.

2. Text-mode

This mode is for modifying the style of text NOT color.

3; italic
4; underline
5; blinking (slow)
6; blinking (fast)
7; reverse
8; hide
9; cross-out
and are almost supported.
For example KDE-Konsole supports 5; but Gnome-Terminal does not and Gnome supports 8; but KDE does not.

3. Foreground mode
This mode is for colorizing the foreground.

4. Background mode
This mode is for colorizing the background.

Colors to current BASH prompt (PS1)

To add colors to the shell prompt use the following export command syntax:
'\e[x;ym $PS1 \e[m'
Where,

\e[ : Start color scheme.
x;y : Color pair to use (x;y)
$PS1 : Your shell prompt variable.
\e[m : Stop color scheme.

Examples using export and tput,
$ export PS1="\e[0;31m[\u@\h \W]\$ \e[m "
$ export PS1="\[$(tput setaf 2)\]\u@\h:\w $ \[$(tput sgr0)\]"

tput

Foreground & background colour commands
tput setab [1-7] # Set the background colour using ANSI escape
tput setaf [1-7] # Set the foreground colour using ANSI escape

Colours are as follows:

No  Colour      #define              R G B

0    black       COLOR_BLACK       0,0,0
1    red           COLOR_RED          1,0,0
2    green       COLOR_GREEN       0,1,0
3    yellow      COLOR_YELLOW    1,1,0
4    blue          COLOR_BLUE        0,0,1
5    magenta   COLOR_MAGENTA  1,0,1
6    cyan         COLOR_CYAN        0,1,1
7    white        COLOR_WHITE      1,1,1

Text mode commands
tput bold    # Select bold mode
tput dim     # Select dim (half-bright) mode
tput smul    # Enable underline mode
tput rmul    # Disable underline mode
tput rev     # Turn on reverse video mode
tput smso    # Enter standout (bold) mode
tput rmso    # Exit standout mode

echo "$(tput setaf 1)Red text $(tput setab 7)and white background$(tput sgr 0)"

Color summary


ANSI Color variables












Code  Effect

  0       reset
  1       lighter than normal
  2       darker than normal
  3       italic
  4       underline
  5       blinking (slow)
  6       blinking (fast)
  7       reverse
  8       hide
  9       cross-out


































# Reset
Color_Off='\033[0m'       # Text Reset

# Regular Colors
Black='\033[0;30m'        # Black
Red='\033[0;31m'          # Red
Green='\033[0;32m'        # Green
Yellow='\033[0;33m'       # Yellow
Blue='\033[0;34m'         # Blue
Purple='\033[0;35m'       # Purple
Cyan='\033[0;36m'         # Cyan
White='\033[0;37m'        # White

# Bold
BBlack='\033[1;30m'       # Black
BRed='\033[1;31m'         # Red
BGreen='\033[1;32m'       # Green
BYellow='\033[1;33m'      # Yellow
BBlue='\033[1;34m'        # Blue
BPurple='\033[1;35m'      # Purple
BCyan='\033[1;36m'        # Cyan
BWhite='\033[1;37m'       # White

# Underline
UBlack='\033[4;30m'       # Black
URed='\033[4;31m'         # Red
UGreen='\033[4;32m'       # Green
UYellow='\033[4;33m'      # Yellow
UBlue='\033[4;34m'        # Blue
UPurple='\033[4;35m'      # Purple
UCyan='\033[4;36m'        # Cyan
UWhite='\033[4;37m'       # White

# Background
On_Black='\033[40m'       # Black
On_Red='\033[41m'         # Red
On_Green='\033[42m'       # Green
On_Yellow='\033[43m'      # Yellow
On_Blue='\033[44m'        # Blue
On_Purple='\033[45m'      # Purple
On_Cyan='\033[46m'        # Cyan
On_White='\033[47m'       # White

# High Intensity
IBlack='\033[0;90m'       # Black
IRed='\033[0;91m'         # Red
IGreen='\033[0;92m'       # Green
IYellow='\033[0;93m'      # Yellow
IBlue='\033[0;94m'        # Blue
IPurple='\033[0;95m'      # Purple
ICyan='\033[0;96m'        # Cyan
IWhite='\033[0;97m'       # White

# Bold High Intensity
BIBlack='\033[1;90m'      # Black
BIRed='\033[1;91m'        # Red
BIGreen='\033[1;92m'      # Green
BIYellow='\033[1;93m'     # Yellow
BIBlue='\033[1;94m'       # Blue
BIPurple='\033[1;95m'     # Purple
BICyan='\033[1;96m'       # Cyan
BIWhite='\033[1;97m'      # White

# High Intensity backgrounds
On_IBlack='\033[0;100m'   # Black
On_IRed='\033[0;101m'     # Red
On_IGreen='\033[0;102m'   # Green
On_IYellow='\033[0;103m'  # Yellow
On_IBlue='\033[0;104m'    # Blue
On_IPurple='\033[0;105m'  # Purple
On_ICyan='\033[0;106m'    # Cyan
On_IWhite='\033[0;107m'   # White


Monday 16 March 2020

Digital clock using watch, echo, banner and tput

Digital Clock

watch

#!/bin/bash
watch -tn 1 date +%T

echo

#!/bin/bash
clear
while [ 1 ] ; do echo -e "$(date +%T)" ; sleep 1; done

banner

#!/bin/bash
clear
while [ 1 ] ; do banner "$(date +%r)" ; sleep 1; clear ; done

while

#!/bin/bash
clear
while :
do
 echo -e "\033[92m $(date '+%r')"
 sleep 1
 clear
done

tput

#!/bin/bash
while sleep 1;do tput sc;tput cup $(($(tput lines)-1)) 1;printf `date +%r`;tput rc;done

#!/bin/bash
clear
while :
do
 ti= date '+%r'
 echo -e -n "\033[7s"
 tput cup 0 69
 echo -n $ti
 echo -e -n "\033[8u"
 sleep 1
 clear
done

Ref:- Google