robots.txt off...

2024-01-18 14:37:17 +01:00
parent 5b01175a64
commit cfcd4d5590
1 changed files with 2 additions and 1 deletions
@@ -1,6 +1,6 @@
 #!/bin/bash

-output="/tmp/website-size.out"
+output="./${!#}.log"
 echo "Analitzant ${!#}..."
 sleep 1s
 echo "Pot tardar una estona..."
@@ -9,6 +9,7 @@ wget \
  --recursive --level=1 \
  --spider --server-response \
  --no-directories --no-parent \
+  -e robots=off \
  --output-file="$output" "$@"

 sleep 1s