--- pandia/crawler	2025/06/27 02:14:47	1.3
+++ pandia/crawler	2025/06/27 16:20:30	1.4
@@ -1,7 +1,7 @@
 #!/usr/bin/env perl
 
 # 
-# $Id: crawler,v 1.3 2025/06/27 02:14:47 snw Exp $
+# $Id: crawler,v 1.4 2025/06/27 16:20:30 snw Exp $
 #  Copyright (C) 2025 Coherent Logic Development LLC
 #
 # Author: Serena Willis <snw@coherent-logic.com>
@@ -9,6 +9,9 @@
 # Licensed AGPL-3.0
 #
 # $Log: crawler,v $
+# Revision 1.4  2025/06/27 16:20:30  snw
+# Add blacklist
+#
 # Revision 1.3  2025/06/27 02:14:47  snw
 # Initial operational capability
 #
@@ -32,9 +35,10 @@ use HTML::TreeBuilder;
 use URI;
 use DBI;
 use WWW::RobotRules;
-my $rules = WWW::RobotRules->new('pandia-crawler/0.0.1');
+use Fcntl qw(:flock);
 use LWP::Simple qw(get);
 
+my $rules = WWW::RobotRules->new('pandia-crawler/0.0.1');
 my $dbh = "";
 my $dsn = "";
 my $skips = 0;
@@ -48,10 +52,10 @@ my $invalid_scheme_skips = 0;
 sub store_url {
     my ($url, $parent) = @_;
 
-    if($url ne "" && length($url) <= 255 && substr($url, 0, 6) ne "mailto") {
-
+    if($url ne "" && length($url) <= 255 && substr($url, 0, 6) ne "mailto" && substr($url, 0, 4) eq "http") {       
+	
 	my $u = URI->new($url);
-	my $domain = $u->host;
+	my $domain = $u->host;	
 	my $scheme = $u->scheme;
 
 	my $sth = $dbh->prepare("INSERT INTO url_domains (url_domain) VALUES (?)");
@@ -118,15 +122,6 @@ sub crawl_url {
 
     my $response = $http->get($url);
 
-    if(not $response->{success}) {
-	print "pandia:  http failure; skipping $url\n";
-	next;
-    }
-	
-    if(exists $response->{redirects}) {
-	print "pandia:  redirects detected; skipping $url\n";
-	next;
-    }
 
     $tree->parse($response->{content});
 
@@ -187,6 +182,9 @@ if($seed ne "") {
     print "[OK]\n";
 }
 else {
+    open my $file, ">", "pandia_crawler.lock" or die $!; 
+    flock $file, LOCK_EX|LOCK_NB or die "Unable to lock file $!";
+
     my $sth = $dbh->prepare("SELECT url FROM crawl_queue");
     $sth->execute();
     my $qlen = $sth->rows;