#!usr/bin/perl use strict; use warnings; use Data::Dumper; use JSON::XS qw(encode_json decode_json); use LWP::Simple; use File::Slurp qw(read_file write_file); use Getopt::Long; my (%db, %ParamHash); my $dbKeysStart = 0; my %config = ( 'api_root_location' => 'https://api.unsplash.com/', 'api_search_root_location' => 'https://api.unsplash.com/search/photos', # 'api_ratelimit_location' => 'https://api.unsplash.com/photos/curated?client_id=45ea109161a2cc78ea845bdca006473edd5a323b2c3802e31b423a672e312ed1&per_page=1', 'api_ratelimit_location' => 'https://api.unsplash.com/photos/?client_id=45ea109161a2cc78ea845bdca006473edd5a323b2c3802e31b423a672e312ed1&per_page=2', 'image_dir' => './images/', 'db_file' => './db/db.dat', 'image_download_size' => 'regular', # raw, full, regular, small, thumb 'credentials' => { 'access_key' => '45ea109161a2cc78ea845bdca006473edd5a323b2c3802e31b423a672e312ed1', 'secret_key' => '1f50382ddcbdb0d47b77faaff870cb0d2894b63589e3aeafbfd2a772d9f4684f', }, 'default_parameters' => { 'search' => { 'per_page' => 10, # Number of items per page. (Optional; default: 10) 'page' => 1, # Page number to retrieve. (Optional; default: 1) # 'query' => ['luxury property', 'luxury real estate', 'villa', 'mansion', 'pool house', 'loft', 'dream house', 'resort luxury'], # Search terms. 'query' => [], # Search terms. 'collections' => '', # Collection ID(‘s) to narrow search. Optional. If multiple, comma-separated. 'orientation' => '', # Filter by photo orientation. Optional. (Valid values: landscape, portrait, squarish) 'order_by' => 'relevant', # How to sort the photos. (Optional; default: relevant). Valid values are latest and relevant. 'content_filter' => 'low', # Limit results by content safety. (Optional; default: low). Valid values are low and high. 'color' => '', # Filter results by color. Optional. Valid values are: black_and_white, black, white, yellow, orange, red, purple, magenta, green, teal, and blue. }, }, ); # &CheckARGV(); &GetCommandFlags(); &UndumpFromFile(); &CheckRateLimit(); # &Search({ 'per_page' => 2 }); &Search(\%ParamHash); &Summary(); sub GetCommandFlags { GetOptions( 'per_page=i' => \$config{'default_parameters'}{'search'}{'per_page'}, 'page=i' => \$config{'default_parameters'}{'search'}{'page'}, 'query=s' => \@{$config{'default_parameters'}{'search'}{'query'}}, 'collections=i' => \$config{'default_parameters'}{'search'}{'collections'}, 'orientation=s' => \$config{'default_parameters'}{'search'}{'orientation'}, 'order_by=s' => \$config{'default_parameters'}{'search'}{'order_by'}, 'content_filter=s' => \$config{'default_parameters'}{'search'}{'content_filter'}, 'color=s' => \$config{'default_parameters'}{'search'}{'color'}, 'image_download_size=s' => \$config{'image_download_size'}, ) or die; if( !@{$config{'default_parameters'}{'search'}{'query'}} ) { die "--query has no arguments."; } } sub CheckARGV { &Delimiter((caller(0))[3]); if (@ARGV) { print Dumper @ARGV; for my $i (0 .. $#ARGV) { my @words = split /=/, $ARGV[$i]; $ParamHash{$words[0]} = $words[1]; } } } sub Summary { &Delimiter((caller(0))[3]); print "$config{'db_file'} has " . scalar(keys(%db)) . " keys (before $dbKeysStart).\n"; } sub AddToDB { # &Delimiter((caller(0))[3]); my ($decoded_json, $i) = @_; $db{@{$$decoded_json{'results'}[$i]}{'id'}} = $$decoded_json{'results'}[$i]; my $json = encode_json \%db; write_file($config{'db_file'}, { binmode => ':raw' }, $json); # print "@{$$decoded_json{'results'}[$i]}{'id'} added to DB file.\n"; } sub UndumpFromFile { &Delimiter((caller(0))[3]); if (-e $config{'db_file'}) { my $json = read_file($config{'db_file'}, { binmode => ':raw' }); if (!$json) { warn "DB file $config{'db_file'} is empty.\n"; return; } %db = %{ decode_json $json }; $dbKeysStart = scalar(keys(%db)); print "INFO: $config{'db_file'} has " . $dbKeysStart . " keys.\n"; } elsif (!-e $config{'db_file'}) { warn "INFO: NO DB file found at $config{'db_file'}. Creating now... "; write_file($config{'db_file'}, ''); print "done.\n"; } } sub CheckRateLimit { &Delimiter((caller(0))[3]); my $res = head($config{'api_ratelimit_location'}); print Dumper $res; print "RATELIMIT: $$res{'_headers'}{'x-ratelimit-remaining'} / $$res{'_headers'}{'x-ratelimit-limit'}\n"; if ($$res{'_headers'}{'x-ratelimit-limit'} - $$res{'_headers'}{'x-ratelimit-remaining'} == 0) { die "Ratelimit reached."; } } sub Search { &Delimiter((caller(0))[3]); my $parameterRef = &CheckHashParameters($_[0]); print Dumper $parameterRef; my $url = &AppendParameter($config{'api_search_root_location'} . "?client_id=$config{'credentials'}{'access_key'}", $parameterRef); my $json = get( $url ); die "Could not get $url" unless defined $json; my $decoded_json = decode_json( $json ); &DownloadImages($decoded_json, $parameterRef); } sub DownloadImages { &Delimiter((caller(0))[3]); my ($decoded_json, $parameterRef) = @_; my $query = $$parameterRef{'query'}; my $download_dir = $config{'image_dir'} . $query; if( !-e $download_dir ) { print "Download directory $download_dir does not exist. Creating now...\n"; mkdir $download_dir or die "Error creating directory: $!"; } else { print "Download directory $download_dir already exists. Nothing to create.\n"; } for (my $i = 0; $i < scalar @{ $$decoded_json{'results'} }; $i++) { if (!exists $db{@{$$decoded_json{'results'}[$i]}{'id'}}) { print "Downloading @{$$decoded_json{'results'}[$i]}{'id'}... "; my $response_code = getstore(@{$$decoded_json{'results'}[$i]}{'urls'}->{$config{'image_download_size'}}, ("$download_dir/" . "@{$$decoded_json{'results'}[$i]}{'id'}.jpg")); if ($response_code == 200) { print "successful.\n"; &AddToDB($decoded_json, $i); } else { die "$response_code received."; } } elsif (exists $db{@{$$decoded_json{'results'}[$i]}{'id'}}) { print "ID @{$$decoded_json{'results'}[$i]}{'id'} already in DB -> Skipping download.\n"; } } } sub AppendParameter { &Delimiter((caller(0))[3]); my ($url, $parameterRef) = @_; for my $key (keys %$parameterRef) { if ($$parameterRef{$key} ne '') { $url .= "&$key=$$parameterRef{$key}"; } } print "URL: $url\n"; return $url } sub CheckHashParameters { &Delimiter((caller(0))[3]); my $parameterRef = shift; if (!defined $parameterRef->{'query'}) { print 'Chosing random query element...'; my $random_element = $config{'default_parameters'}{'search'}{'query'}[rand @{$config{'default_parameters'}{'search'}{'query'}}]; print " '$random_element'\n"; $parameterRef->{'query'} = $random_element; } for my $key (keys %$parameterRef) { if (!defined $config{'default_parameters'}{'search'}{$key}) { print "Parameter '$key' is not valid. Allowed parameter are: "; print "'$_' " for keys(%{$config{'default_parameters'}{'search'}}); print "\n"; die; } } for my $key (keys(%{$config{'default_parameters'}{'search'}})) { if (!defined $parameterRef->{$key}) { $parameterRef->{$key} = $config{'default_parameters'}{'search'}{$key}; } } return $parameterRef; } sub Delimiter { my $SubName = shift; print "\n" . "-" x 80 . "\nSUB " . $SubName . "\n" . '-' x 80 . "\n"; }