fix: remove top level query string

- Closes #15
2020-05-15 11:14:57 +01:00 · 2020-05-15 11:14:57 +01:00 · 8a026ce356
parent baebc326f1
commit 8a026ce356
1 changed files with 5 additions and 0 deletions
--- a/script.sh
+++ b/script.sh
@ -35,6 +35,9 @@ sort -u > "urlhaus.txt"
 cat "urlhaus.txt" | \
 cut -f 1 -d "/" | \
 cut -f 1 -d ":" | \
+# Remove invalid query string, see #15
+cut -f 1 -d "?" | \
+grep -vF "?" | \
 sort -u > "urlhaus-domains.txt"

 ## Parse online URLs only
@ -48,6 +51,8 @@ sort -u > "urlhaus-online.txt"
 cat "urlhaus-online.txt" | \
 cut -f 1 -d "/" | \
 cut -f 1 -d ":" | \
+cut -f 1 -d "?" | \
+grep -vF "?" | \
 sort -u > "urlhaus-domains-online.txt"