Pejman Moghadam / Scripts

Download entire commandlinefu.com in a single txt file

Public domain

---

#!/bin/bash

DATE=$(date +'%F-%H-%M-%S')
time {
  COUNT=$(
    curl http://www.commandlinefu.com/commands/browse | 
    grep "Terminal - All commands" | 
    awk ' {
      for(i=1;i<=NF;i++)
        if($i ~ /[0-9]/) {
          gsub(/','/, "", $i);
          print $i;
        }
    }'
  )
  TOTAL=$(( $COUNT / 25 + 1 ))
  NUM=1
  for I in $(seq 0 25 $COUNT); do 
    echo; echo; echo "=== Download: $NUM / $TOTAL ==="
    curl http://www.commandlinefu.com/commands/browse/sort-by-votes/plaintext/$I >> commandlinefu-${DATE}.txt
    NUM=$(( $NUM + 1 ))
  done
}

---

BY: Pejman Moghadam
TAG: bash-script, bash, commandlinefu, curl, awk
DATE: 2011-12-08 17:10:03

---

Pejman Moghadam / Scripts [ TXT ]