Pejman Moghadam / Slackware

Slackware 12.2 - Squid 2.7 / COSS

Public domain


Calculations

cache_mem(MB) = RAM(MB) / 2
cache_mem(MB) = (2 * 1024) / 2 = 1024 MB
cache_mem 1024 MB

Total cache_dir(GB) = cache_mem(MB) / 10
Total cache_dir(GB) = 1024 / 10 = 102 GB

each cache_dir(MB) = ( Total cache_dir / 4) * 1024
each cache_dir(MB) = (102 / 4) * 1024 = 26112 MB

empty on each partition = partition size * 0.3
empty on each partition = 66 GB * 0.3 = 19.8 GB

max usage on each partition MB = (partition size(GB) - empty on each partition(GB)) *1024
max usage on each partition = ( 66 - 19.8 ) * 1024= 47308 

max usage on each partition > each cache_dir : OK
47308 > 26112 : OK ==> each cache_dir(MB) = 26112 MB

block-size=512  -> max cache_dir 8192 MB
block-size=1024 -> max cache_dir 16384 MB
block-size=2048 -> max cache_dir 32768 MB
block-size=4096 -> max cache_dir 65536 MB
block-size=8192 -> max cache_dir 131072 MB

block-size = cache_dir(MB) / 16
block-size = 26112 / 16 = 1632
1024 < 1632 < 2048
block-size = 2048

cache_dir 1024 MB
cache_dir coss /cache/1 26112 max-size=262144  block-size=2048
cache_dir coss /cache/2 26112 max-size=524288  block-size=2048
cache_dir coss /cache/3 26112 max-size=2097152 block-size=2048
cache_dir coss /cache/4 26112 max-size=8388608 block-size=2048

Resource : http://etutorials.org/Server+Administration/Squid.+The+definitive+guide/Chapter+8.+Advanced+Disk+Cache+Topics/8.6+The+coss+Storage+Scheme/
Resource : http://wiki.squid-cache.org/Features/CyclicObjectStorageSystem

Partitions & memory

# cat /etc/fstab
/dev/sda1        swap             swap        defaults         0   0
/dev/sda2        /                reiserfs    defaults         1   1
/dev/sda5        /cache/1         ext2        defaults,noatime 1   2
/dev/sda6        /cache/2         ext2        defaults,noatime 1   2
/dev/sda7        /cache/3         ext2        defaults,noatime 1   2
/dev/sda8        /cache/4         ext2        defaults,noatime 1   2
/dev/cdrom       /mnt/cdrom       auto        noauto,owner,ro  0   0
/dev/fd0         /mnt/floppy      auto        noauto,owner     0   0
devpts           /dev/pts         devpts      gid=5,mode=620   0   0
proc             /proc            proc        defaults         0   0
tmpfs            /dev/shm         tmpfs       defaults         0   0


# df -hT
Filesystem    Type    Size  Used Avail Use% Mounted on
/dev/root reiserfs     19G  3.8G   15G  21% /
/dev/sda5     ext2     69G   52M   66G   1% /cache/1
/dev/sda6     ext2     69G   52M   66G   1% /cache/2
/dev/sda7     ext2     69G   52M   66G   1% /cache/3
/dev/sda8     ext2     69G   52M   66G   1% /cache/4
tmpfs        tmpfs   1011M     0 1011M   0% /dev/shm


# free -m
             total       used       free     shared    buffers     cached
Mem:          2021        280       1740          0        100        100
-/+ buffers/cache:         80       1940
Swap:          250          0        250

Download and Install

# vi /usr/include/bits/typesizes.h
    #define __FD_SETSIZE            8192
# cd /usr/src/
# wget http://www.squid-cache.org/Versions/v2/2.7/squid-2.7.STABLE6.tar.gz 
# tar zxf squid-2.7.STABLE6.tar.gz 
# cd squid-2.7.STABLE6
# ulimit -HSn 8192
# ./configure \
    --disable-internal-dns \
    --enable-forward-log \
    --enable-follow-x-forwarded-for \
    --enable-snmp \
    --enable-linux-netfilter \
    --enable-http-violations \
    --enable-delay-pools \
    --enable-storeio=diskd,aufs,ufs,coss \
    --with-coss-membuf-size=8388608 \
    --with-large-files \
    --enable-large-cache-files \
    --with-maxfd=8192 \
    --enable-async-io=64 \
    --enable-removal-policies=lru,heap \
    --enable-useragent-log \
    --enable-referer-log \
    --enable-err-languages=English \
    --enable-default-err-language=English
# make && make install
# cp /usr/local/squid/etc/squid.conf{,.bak}
# egrep -v '^#|^ *$' /usr/local/squid/etc/squid.conf.bak > /usr/local/squid/etc/squid.conf

/usr/local/squid/etc/squid.conf

acl all src all
acl manager proto cache_object
acl localhost src 127.0.0.1/32
acl to_localhost dst 127.0.0.0/8
#acl localnet src 10.0.0.0/8    # RFC1918 possible internal network
#acl localnet src 172.16.0.0/12 # RFC1918 possible internal network
#acl localnet src 192.168.0.0/16        # RFC1918 possible internal network
acl SSL_ports port 443
acl Safe_ports port 80          # http
acl Safe_ports port 21          # ftp
acl Safe_ports port 443         # https
acl Safe_ports port 70          # gopher
acl Safe_ports port 210         # wais
acl Safe_ports port 1025-65535  # unregistered ports
acl Safe_ports port 280         # http-mgmt
acl Safe_ports port 488         # gss-http
acl Safe_ports port 591         # filemaker
acl Safe_ports port 777         # multiling http
acl CONNECT method CONNECT
http_access allow manager localhost
http_access deny manager
http_access deny !Safe_ports
http_access deny CONNECT !SSL_ports
acl our_networks src 85.185.14.0/25 85.185.16.192/26 217.218.228.192/26
http_access allow our_networks
http_access allow localhost
#http_access allow localnet
http_access deny all
#icp_access allow localnet
icp_access deny all
http_port 3128 transparent
hierarchy_stoplist cgi-bin ? dll aspx
cache_mem 1024 MB
maximum_object_size_in_memory 64 KB
cache_replacement_policy heap LFUDA
cache_dir coss /cache/1 26112 max-size=262144  block-size=2048
cache_dir coss /cache/2 26112 max-size=524288  block-size=2048
cache_dir coss /cache/3 26112 max-size=2097152 block-size=2048
cache_dir coss /cache/4 26112 max-size=8388608 block-size=2048
maximum_object_size 8192 KB
cache_swap_low 95
cache_swap_high 100
logformat squid  %tl.%03tu %6tr %>a %Ss/%03Hs %<st %rm %ru %un %Sh/%<A %mt
access_log /usr/local/squid/var/logs/access.log squid
cache_swap_state /usr/local/squid/var/logs/cache_swap_log
logfile_rotate 0
refresh_pattern http://.*\.windowsupdate\.microsoft\.com/ 0 80% 20160 reload-into-ims
refresh_pattern http://office\.microsoft\.com/ 0 80% 20160 reload-into-ims
refresh_pattern http://windowsupdate\.microsoft\.com/ 0 80% 20160 reload-into-ims
refresh_pattern http://w?xpsp[0-9]\.microsoft\.com/ 0 80% 20160 reload-into-ims
refresh_pattern http://w2ksp[0-9]\.microsoft\.com/ 0 80% 20160 reload-into-ims
refresh_pattern http://download\.microsoft\.com/ 0 80% 20160 reload-into-ims
refresh_pattern http://download\.macromedia\.com/ 0 80% 20160 reload-into-ims
refresh_pattern ftp://ftp\.nai\.com/ 0 80% 20160 reload-into-ims
refresh_pattern http://ftp\.software\.ibm\.com/ 0 80% 20160 reload-into-ims
refresh_pattern         cgi-bin         1 20% 2
refresh_pattern         \.asp$          1 20% 2
refresh_pattern         \.acgi$         1 20% 2
refresh_pattern         \.cgi$          1 20% 2
refresh_pattern         \.pl$           1 20% 2
refresh_pattern         \.shtml$        1 20% 2
refresh_pattern         \.php3$         1 20% 2
refresh_pattern         \?              1 20% 2
refresh_pattern         \.gif$          10080   90%     43200 reload-into-ims
refresh_pattern         \.jpg$          10080   90%     43200 reload-into-ims
refresh_pattern         \.bom\.gov\.au     30   20%       120 reload-into-ims
refresh_pattern         \.html$           480   50%     22160 reload-into-ims
refresh_pattern         \.htm$            480   50%     22160 reload-into-ims
refresh_pattern         \.class$        10080   90%     43200 reload-into-ims
refresh_pattern         \.zip$          10080   90%     43200 reload-into-ims
refresh_pattern         \.jpeg$         10080   90%     43200 reload-into-ims
refresh_pattern         \.mid$          10080   90%     43200 reload-into-ims
refresh_pattern         \.shtml$          480   50%     22160 reload-into-ims
refresh_pattern         \.exe$          10080   90%     43200 reload-into-ims
refresh_pattern         \.thm$          10080   90%     43200 reload-into-ims
refresh_pattern         \.wav$          10080   90%     43200 reload-into-ims
refresh_pattern         \.txt$          10080   90%     43200 reload-into-ims
refresh_pattern         \.cab$          10080   90%     43200 reload-into-ims
refresh_pattern         \.au$           10080   90%     43200 reload-into-ims
refresh_pattern         \.mov$          10080   90%     43200 reload-into-ims
refresh_pattern         \.xbm$          10080   90%     43200 reload-into-ims
refresh_pattern         \.ram$          10080   90%     43200 reload-into-ims
refresh_pattern         \.avi$          10080   90%     43200 reload-into-ims
refresh_pattern         \.chtml$          480   50%     22160 reload-into-ims
refresh_pattern         \.thb$          10080   90%     43200 reload-into-ims
refresh_pattern         \.dcr$          10080   90%     43200 reload-into-ims
refresh_pattern         \.bmp$          10080   90%     43200 reload-into-ims
refresh_pattern         \.phtml$          480   50%     22160 reload-into-ims
refresh_pattern         \.mpg$          10080   90%     43200 reload-into-ims
refresh_pattern         \.pdf$          10080   90%     43200 reload-into-ims
refresh_pattern         \.art$          10080   90%     43200 reload-into-ims
refresh_pattern         \.swf$          10080   90%     43200 reload-into-ims
refresh_pattern         \.mp3$          10080   90%     43200 reload-into-ims
refresh_pattern         \.ra$           10080   90%     43200 reload-into-ims
refresh_pattern         \.spl$          10080   90%     43200 reload-into-ims
refresh_pattern         \.viv$          10080   90%     43200 reload-into-ims
refresh_pattern         \.doc$          10080   90%     43200 reload-into-ims
refresh_pattern         \.gz$           10080   90%     43200 reload-into-ims
refresh_pattern         \.Z$            10080   90%     43200 reload-into-ims
refresh_pattern         \.tgz$          10080   90%     43200 reload-into-ims
refresh_pattern         \.tar$          10080   90%     43200 reload-into-ims
refresh_pattern         \.vrm$          10080   90%     43200 reload-into-ims
refresh_pattern         \.vrml$         10080   90%     43200 reload-into-ims
refresh_pattern         \.aif$          10080   90%     43200 reload-into-ims
refresh_pattern         \.aifc$         10080   90%     43200 reload-into-ims
refresh_pattern         \.aiff$         10080   90%     43200 reload-into-ims
refresh_pattern         \.arj$          10080   90%     43200 reload-into-ims
refresh_pattern         \.c$            10080   90%     43200 reload-into-ims
refresh_pattern         \.cpt$          10080   90%     43200 reload-into-ims
refresh_pattern         \.dir$          10080   90%     43200 reload-into-ims
refresh_pattern         \.dxr$          10080   90%     43200 reload-into-ims
refresh_pattern         \.hqx$          10080   90%     43200 reload-into-ims
refresh_pattern         \.jpe$          10080   90%     43200 reload-into-ims
refresh_pattern         \.lha$          10080   90%     43200 reload-into-ims
refresh_pattern         \.lzh$          10080   90%     43200 reload-into-ims
refresh_pattern         \.midi$         10080   90%     43200 reload-into-ims
refresh_pattern         \.movie$        10080   90%     43200 reload-into-ims
refresh_pattern         \.mp2$          10080   90%     43200 reload-into-ims
refresh_pattern         \.mpe$          10080   90%     43200 reload-into-ims
refresh_pattern         \.mpeg$         10080   90%     43200 reload-into-ims
refresh_pattern         \.mpga$         10080   90%     43200 reload-into-ims
refresh_pattern         \.pl$           10080   90%     43200 reload-into-ims
refresh_pattern         \.ppt$          10080   90%     43200 reload-into-ims
refresh_pattern         \.ps$           10080   90%     43200 reload-into-ims
refresh_pattern         \.qt$           10080   90%     43200 reload-into-ims
refresh_pattern         \.qtm$          10080   90%     43200 reload-into-ims
refresh_pattern         \.ras$          10080   90%     43200 reload-into-ims
refresh_pattern         \.sea$          10080   90%     43200 reload-into-ims
refresh_pattern         \.sit$          10080   90%     43200 reload-into-ims
refresh_pattern         \.tif$          10080   90%     43200 reload-into-ims
refresh_pattern         \.tiff$         10080   90%     43200 reload-into-ims
refresh_pattern         \.snd$          10080   90%     43200 reload-into-ims
refresh_pattern         \.wrl$          10080   90%     43200 reload-into-ims
refresh_pattern         ^ftp:           1440    60%     22160
refresh_pattern         ^gopher:        1440    20%     1440
refresh_pattern         -i (cgi-bin|\?) 0       0%      0
refresh_pattern         .               480     50%     22160 reload-into-ims
acl shoutcast rep_header X-HTTP09-First-Line ^ICY.[0-9]
upgrade_http0.9 deny shoutcast
quick_abort_min 32 KB
quick_abort_max 32 KB
quick_abort_pct 95
negative_ttl 3 minutes
positive_dns_ttl 15 hours
request_header_max_size 100 KB
acl apache rep_header Server ^Apache
broken_vary_encoding allow apache
cache_mgr Pejman_Moghadam@yahoo.com
visible_hostname CacheServer
acl snmppublic snmp_community public
snmp_access allow snmppublic localhost
snmp_access deny all
dns_children 100
coredump_dir /usr/local/squid/var/cache
pipeline_prefetch on

/etc/rc.d/rc.squid

#!/bin/sh
#
# /etc/rc.d/rc.squid
#
# Start/stop/restart the Squid web caching server.
#
# To make Squid start automatically at boot, make this
# file executable: chmod 755 /etc/rc.d/rc.squid
#

start()
{
  echo -n 'Starting Squid . . . '

  PROCESS=$(ps -A | egrep ' squid$')
  if [ "$PROCESS" == "" ]; then
    if [ -f /usr/local/squid/var/logs/squid.pid ] ; then
      rm /usr/local/squid/var/logs/squid.pid
    fi
  fi
  echo "32768 61000" > /proc/sys/net/ipv4/ip_local_port_range
  ulimit -HSn 8192
  /usr/local/squid/sbin/squid -D

  echo "Ok"
}

stop()
{
  echo 'Stoping Squid'

  /usr/local/squid/sbin/squid -k shutdown
  time=0
  while [ $time != "300" ] ; do
    time=`expr $time + 1`
    echo -n $time
    if [ ! -f /usr/local/squid/var/logs/squid.pid ] ; then
      break
    else
      echo -n "."
    fi
    sleep 1
  done

  echo ". .Ok"
}

reload()
{
  echo 'Reloading Squid'
  /usr/local/squid/sbin/squid -k reconfigure
  echo "Ok"
}

case "$1" in
  'start')
    start
    ;;

  'stop')
    stop
    ;;

  'restart')
    stop
    start
    ;;

  'rotate')
    echo -n 'Rotating Squid log files . . . '
    /usr/local/squid/sbin/squid -k rotate
    echo "Ok"
    ;;

   'reload')
     reload
     ;;

  *)
    echo "usage $0 start|stop|restart|reload|rotate"
    ;;

esac

/etc/rc.d/rc.local

# Start Squid Cache server:
if [ -x /etc/rc.d/rc.squid ]; then
  /etc/rc.d/rc.squid start
fi

/etc/rc.d/rc.local_shutdown

#!/bin/bash
# Stop Squid Cache server:
if [ -x /etc/rc.d/rc.squid ]; then
  /etc/rc.d/rc.squid stop
fi

/etc/logrotate.d/squid

/usr/local/squid/var/logs/access.log {
  daily
  rotate 180
  start 1
  copytruncate
  compress
  compresscmd /usr/bin/bzip2
  compressext .bz2
  compressoptions -sq9
  dateext
  notifempty
  missingok
}

/usr/local/squid/var/logs/cache.log /usr/local/squid/var/logs/store.log {
  daily
  rotate 30
  start 1
  copytruncate
  compress
  compresscmd /usr/bin/bzip2
  compressext .bz2
  compressoptions -sq9
  dateext
  notifempty
  missingok
  sharedscripts
  postrotate
    /usr/local/squid/sbin/squid -k rotate
  endscript
}

First time lunch

mkdir /usr/local/squid/var/cache
mkdir -p /cache/{1,2,3,4}
chown -R nobody:nobody /cache
chown -R nobody:nobody /usr/local/squid/var/logs
chmod +x /etc/rc.d/rc.local_shutdown
chmod +x /etc/rc.d/rc.squid
/usr/local/squid/sbin/squid -z
/etc/rc.d/rc.squid start

BY: Pejman Moghadam
TAG: squid, coss, fstab
DATE: 2009-05-26 22:05:30


Pejman Moghadam / Slackware [ TXT ]