pandia/crawler - diff

Return to crawler CVS log

Up to [Coherent Logic Development] / pandia

version 1.7, 2025/07/02 15:03:05	version 1.9, 2025/07/05 15:27:53
Line 9	Line 9
# Licensed AGPL-3.0	# Licensed AGPL-3.0
#	#
# $Log$	# $Log$
	# Revision 1.9 2025/07/05 15:27:53 snw
	# Update
	#
	# Revision 1.8 2025/07/02 15:14:44 snw
	# Fix bug in restricted mode
	#
# Revision 1.7 2025/07/02 15:03:05 snw	# Revision 1.7 2025/07/02 15:03:05 snw
# Add support for restricted mode	# Add support for restricted mode
#	#
Line 209 $dbpw = $cfg->val($profile, 'dbpass');	Line 215 $dbpw = $cfg->val($profile, 'dbpass');
$tmp = $cfg->val($profile, 'allowed_tlds');	$tmp = $cfg->val($profile, 'allowed_tlds');

if($tmp ne '*') {	if($tmp ne '*') {
$mode = 'restrictive';	$mode = 'restricted';
@allowed_tlds = split(',', $tmp);	@allowed_tlds = split(',', $tmp);
print "pandia: crawler restricted to these TLDs: ";	print "pandia: crawler restricted to these TLDs: ";
foreach (@allowed_tlds) {	foreach (@allowed_tlds) {
Line 254 else {	Line 260 else {
my $qlen = $sth->rows;	my $qlen = $sth->rows;


print "pandia: crawling queue with length of $qlen to a maximum depth of $maxdepth";	print "pandia: crawling queue with length of $qlen to a maximum depth of $maxdepth...";
sleep 1;	sleep 1;
while (my @row = $sth->fetchrow_array()) {	while (my @row = $sth->fetchrow_array()) {
my $url = @row[0];	my $url = @row[0];