robots.txt off...

website-size.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-output="/tmp/website-size.out"
+output="./${!#}.log"
 echo "Analitzant ${!#}..."
 sleep 1s
 echo "Pot tardar una estona..."
@@ -9,6 +9,7 @@ wget \
   --recursive --level=1 \
   --spider --server-response \
   --no-directories --no-parent \
+  -e robots=off \
   --output-file="$output" "$@"
 
 sleep 1s