Monday, 30 March 2020

Bash script to remove duplicate files

Identify the duplicate files and remove.

vi remove_duplicate.sh
#!/bin/bash
ls -ls --time-style=long-iso | awk 'BEGIN {
 getline; getline;
 name1=$9; size=$5
}
{
 name2=$9;
 if (size==$5)
 {
  "md5sum "name1 | getline; csum1=$1;
  "md5sum "name2 | getline; csum2=$1;
  if ( csum1==csum2 )
        {
         print name1; print name2;
        }
 };

 size=$5; name1=name2;
}' | sort -u > duplicate_files

cat duplicate_files | xargs -I {} md5sum {} | sort | uniq -w 32 | awk '{ print "^"$2"$" }' | sort -u > duplicate_sample

if [ -s duplicate_sample ]
 then
  echo Removing...
  comm duplicate_files duplicate_sample -2 -3 | tee /dev/stderr | xargs rm -rf
  echo Removed duplicate files successfully.
 else
 echo No duplicate files found.
fi

Friday, 27 March 2020

20 Examples of Linux xargs Command


 xargs Command usage

man xargs
xargs --version

ls -l /usr/local/src/ | xargs                #single line output
cat example.txt | xargs -n 7
xargs -a example.txt
echo 'one two three' | xargs mkdir #creating three dirs
echo 'one two three' | xargs -p touch #creating three files

printf "one  three  two" | xargs -i touch {}.txt #add extention
echo "Text1XText2XText3XText4" | xargs -d X -n 2
printf %s\\n {1..30} | xargs -n 5 -P 8                     #executing commands in parallel
cat args.txt | xargs -n 2 '/usr/local/src/cecho.sh'

cat args.txt | xargs -I {} ./cecho.sh -p {} -l
echo "file1 file2 file3" | xargs -t -I % sh -c '{ touch %; ls -l %; }' #multiple commands
find ./*.txt -print0 | xargs -0 -n 1 -P 3 bzip2    #bzip2 processes in parallel
ls "b.txt" | xargs -n 1 sed -i "s/color/colour/g" #replace all occurrences

cut -d: -f1 < /etc/passwd | sort | xargs #compact list of all Linux users
ls *digital* | xargs wc                             #number of lines/words/characters in each file
echo dir1/ dir2/ dir3/ | xargs -n 1 cp -v dir/test.txt #copy a file to multiple dirs at once
find . -type f -name “*.java” | xargs tar cvf myfile.tar
find / -type f -name "*.sh" -print0 | xargs -0 wc -l #cout no of lines

find . -type f -name "*.txt" -print0 | xargs -0 rm -f                 #remove files
find . -type f -name "b.txt" -print0 | xargs -0 -p -n 1  rm -rf   #prompt before execution
find /tmp -mtime +7 | xargs rm

find . -type f -not -name '*.txt' -print0 | xargs -0 -I {} rm -v {} #remove except extension
find . -type d -name "abb" -print0 | xargs -0 rm -v -rf "{}"        #remove directory
echo 'one two three' | xargs -t rm -rf                               #remove three files/dirs

Ref:- linuxtechi.com shapeshed.com tecmint.com