init
This commit is contained in:
216
unsplash (SFConflict zino@zinomedia.de 2020-10-30-17-18-57).pl
Executable file
216
unsplash (SFConflict zino@zinomedia.de 2020-10-30-17-18-57).pl
Executable file
@@ -0,0 +1,216 @@
|
||||
#!usr/bin/perl
|
||||
use strict;
|
||||
use warnings;
|
||||
use Data::Dumper;
|
||||
use JSON::XS qw(encode_json decode_json);
|
||||
use LWP::Simple;
|
||||
use File::Slurp qw(read_file write_file);
|
||||
use Getopt::Long;
|
||||
|
||||
my (%db, %ParamHash);
|
||||
my $dbKeysStart = 0;
|
||||
my %config = (
|
||||
'api_root_location' => 'https://api.unsplash.com/',
|
||||
'api_search_root_location' => 'https://api.unsplash.com/search/photos',
|
||||
# 'api_ratelimit_location' => 'https://api.unsplash.com/photos/curated?client_id=45ea109161a2cc78ea845bdca006473edd5a323b2c3802e31b423a672e312ed1&per_page=1',
|
||||
'api_ratelimit_location' => 'https://api.unsplash.com/photos/?client_id=45ea109161a2cc78ea845bdca006473edd5a323b2c3802e31b423a672e312ed1&per_page=2',
|
||||
'image_dir' => './images/',
|
||||
'db_file' => './db/db.dat',
|
||||
'image_download_size' => 'regular', # raw, full, regular, small, thumb
|
||||
'credentials' => {
|
||||
'access_key' => '45ea109161a2cc78ea845bdca006473edd5a323b2c3802e31b423a672e312ed1',
|
||||
'secret_key' => '1f50382ddcbdb0d47b77faaff870cb0d2894b63589e3aeafbfd2a772d9f4684f',
|
||||
},
|
||||
'default_parameters' => {
|
||||
'search' => {
|
||||
'per_page' => 10, # Number of items per page. (Optional; default: 10)
|
||||
'page' => 1, # Page number to retrieve. (Optional; default: 1)
|
||||
# 'query' => ['luxury property', 'luxury real estate', 'villa', 'mansion', 'pool house', 'loft', 'dream house', 'resort luxury'], # Search terms.
|
||||
'query' => [], # Search terms.
|
||||
'collections' => '', # Collection ID(‘s) to narrow search. Optional. If multiple, comma-separated.
|
||||
'orientation' => '', # Filter by photo orientation. Optional. (Valid values: landscape, portrait, squarish)
|
||||
'order_by' => 'relevant', # How to sort the photos. (Optional; default: relevant). Valid values are latest and relevant.
|
||||
'content_filter' => 'low', # Limit results by content safety. (Optional; default: low). Valid values are low and high.
|
||||
'color' => '', # Filter results by color. Optional. Valid values are: black_and_white, black, white, yellow, orange, red, purple, magenta, green, teal, and blue.
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
# &CheckARGV();
|
||||
&GetCommandFlags();
|
||||
&UndumpFromFile();
|
||||
&CheckRateLimit();
|
||||
# &Search({ 'per_page' => 2 });
|
||||
&Search(\%ParamHash);
|
||||
&Summary();
|
||||
|
||||
|
||||
sub GetCommandFlags {
|
||||
GetOptions(
|
||||
'per_page=i' => \$config{'default_parameters'}{'search'}{'per_page'},
|
||||
'page=i' => \$config{'default_parameters'}{'search'}{'page'},
|
||||
'query=s' => \@{$config{'default_parameters'}{'search'}{'query'}},
|
||||
'collections=i' => \$config{'default_parameters'}{'search'}{'collections'},
|
||||
'orientation=s' => \$config{'default_parameters'}{'search'}{'orientation'},
|
||||
'order_by=s' => \$config{'default_parameters'}{'search'}{'order_by'},
|
||||
'content_filter=s' => \$config{'default_parameters'}{'search'}{'content_filter'},
|
||||
'color=s' => \$config{'default_parameters'}{'search'}{'color'},
|
||||
'image_download_size=s' => \$config{'image_download_size'},
|
||||
) or die;
|
||||
|
||||
|
||||
if( !@{$config{'default_parameters'}{'search'}{'query'}} ) {
|
||||
die "--query has no arguments.";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
sub CheckARGV {
|
||||
&Delimiter((caller(0))[3]);
|
||||
|
||||
if (@ARGV) {
|
||||
print Dumper @ARGV;
|
||||
|
||||
for my $i (0 .. $#ARGV) {
|
||||
my @words = split /=/, $ARGV[$i];
|
||||
$ParamHash{$words[0]} = $words[1];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub Summary {
|
||||
&Delimiter((caller(0))[3]);
|
||||
|
||||
print "$config{'db_file'} has " . scalar(keys(%db)) . " keys (before $dbKeysStart).\n";
|
||||
}
|
||||
|
||||
sub AddToDB {
|
||||
# &Delimiter((caller(0))[3]);
|
||||
my ($decoded_json, $i) = @_;
|
||||
|
||||
$db{@{$$decoded_json{'results'}[$i]}{'id'}} = $$decoded_json{'results'}[$i];
|
||||
my $json = encode_json \%db;
|
||||
write_file($config{'db_file'}, { binmode => ':raw' }, $json);
|
||||
# print "@{$$decoded_json{'results'}[$i]}{'id'} added to DB file.\n";
|
||||
}
|
||||
|
||||
sub UndumpFromFile {
|
||||
&Delimiter((caller(0))[3]);
|
||||
|
||||
if (-e $config{'db_file'}) {
|
||||
my $json = read_file($config{'db_file'}, { binmode => ':raw' });
|
||||
if (!$json) {
|
||||
warn "DB file $config{'db_file'} is empty.\n";
|
||||
return;
|
||||
}
|
||||
%db = %{ decode_json $json };
|
||||
$dbKeysStart = scalar(keys(%db));
|
||||
print "INFO: $config{'db_file'} has " . $dbKeysStart . " keys.\n";
|
||||
}
|
||||
elsif (!-e $config{'db_file'}) {
|
||||
warn "INFO: NO DB file found at $config{'db_file'}. Creating now... ";
|
||||
write_file($config{'db_file'}, '');
|
||||
print "done.\n";
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
sub CheckRateLimit {
|
||||
&Delimiter((caller(0))[3]);
|
||||
|
||||
my $res = head($config{'api_ratelimit_location'});
|
||||
print Dumper $res;
|
||||
print "RATELIMIT: $$res{'_headers'}{'x-ratelimit-remaining'} / $$res{'_headers'}{'x-ratelimit-limit'}\n";
|
||||
if ($$res{'_headers'}{'x-ratelimit-limit'} - $$res{'_headers'}{'x-ratelimit-remaining'} == 0) {
|
||||
die "Ratelimit reached.";
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
sub Search {
|
||||
&Delimiter((caller(0))[3]);
|
||||
my $parameterRef = &CheckHashParameters($_[0]);
|
||||
print Dumper $parameterRef;
|
||||
|
||||
my $url = &AppendParameter($config{'api_search_root_location'} . "?client_id=$config{'credentials'}{'access_key'}", $parameterRef);
|
||||
my $json = get( $url );
|
||||
die "Could not get $url" unless defined $json;
|
||||
my $decoded_json = decode_json( $json );
|
||||
&DownloadImages($decoded_json, $parameterRef);
|
||||
}
|
||||
|
||||
sub DownloadImages {
|
||||
&Delimiter((caller(0))[3]);
|
||||
my ($decoded_json, $parameterRef) = @_;
|
||||
|
||||
my $query = $$parameterRef{'query'};
|
||||
my $download_dir = $config{'image_dir'} . $query;
|
||||
if( !-e $download_dir ) {
|
||||
print "Download directory $download_dir does not exist. Creating now...\n";
|
||||
mkdir $download_dir or die "Error creating directory: $!";
|
||||
}
|
||||
else {
|
||||
print "Download directory $download_dir already exists. Nothing to create.\n";
|
||||
}
|
||||
|
||||
for (my $i = 0; $i < scalar @{ $$decoded_json{'results'} }; $i++) {
|
||||
if (!exists $db{@{$$decoded_json{'results'}[$i]}{'id'}}) {
|
||||
print "Downloading @{$$decoded_json{'results'}[$i]}{'id'}... ";
|
||||
my $response_code = getstore(@{$$decoded_json{'results'}[$i]}{'urls'}->{$config{'image_download_size'}}, ("$download_dir/" . "@{$$decoded_json{'results'}[$i]}{'id'}.jpg"));
|
||||
if ($response_code == 200) {
|
||||
print "successful.\n";
|
||||
&AddToDB($decoded_json, $i);
|
||||
}
|
||||
else {
|
||||
die "$response_code received.";
|
||||
}
|
||||
}
|
||||
elsif (exists $db{@{$$decoded_json{'results'}[$i]}{'id'}}) {
|
||||
print "ID @{$$decoded_json{'results'}[$i]}{'id'} already in DB -> Skipping download.\n";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sub AppendParameter {
|
||||
&Delimiter((caller(0))[3]);
|
||||
my ($url, $parameterRef) = @_;
|
||||
|
||||
for my $key (keys %$parameterRef) {
|
||||
if ($$parameterRef{$key} ne '') {
|
||||
$url .= "&$key=$$parameterRef{$key}";
|
||||
}
|
||||
}
|
||||
print "URL: $url\n";
|
||||
return $url
|
||||
}
|
||||
|
||||
sub CheckHashParameters {
|
||||
&Delimiter((caller(0))[3]);
|
||||
my $parameterRef = shift;
|
||||
|
||||
if (!defined $parameterRef->{'query'}) {
|
||||
print 'Chosing random query element...';
|
||||
my $random_element = $config{'default_parameters'}{'search'}{'query'}[rand @{$config{'default_parameters'}{'search'}{'query'}}];
|
||||
print " '$random_element'\n";
|
||||
$parameterRef->{'query'} = $random_element;
|
||||
}
|
||||
for my $key (keys %$parameterRef) {
|
||||
if (!defined $config{'default_parameters'}{'search'}{$key}) {
|
||||
print "Parameter '$key' is not valid. Allowed parameter are: ";
|
||||
print "'$_' " for keys(%{$config{'default_parameters'}{'search'}});
|
||||
print "\n";
|
||||
die;
|
||||
}
|
||||
}
|
||||
for my $key (keys(%{$config{'default_parameters'}{'search'}})) {
|
||||
if (!defined $parameterRef->{$key}) {
|
||||
$parameterRef->{$key} = $config{'default_parameters'}{'search'}{$key};
|
||||
}
|
||||
}
|
||||
return $parameterRef;
|
||||
}
|
||||
|
||||
sub Delimiter {
|
||||
my $SubName = shift;
|
||||
print "\n" . "-" x 80 . "\nSUB " . $SubName . "\n" . '-' x 80 . "\n";
|
||||
}
|
||||
Reference in New Issue
Block a user