More work on S3 support

Author: jcameron

backups-lib.pl

@@ -6625,9 +6625,11 @@ sub rename_backup_owner
 sub list_all_s3_accounts
 {
 local @rv;
-if (&can_use_cloud("s3") && $config{'s3_akey'} && $config{'s3_skey'}) {
-	push(@rv, [ $config{'s3_akey'}, $config{'s3_skey'},
-		    $config{'s3_endpoint'} ]);
+if (&can_cloud_providers()) {
+	foreach my $s3 (&list_s3_accounts()) {
+		push(@rv, [ $s3->{'access'}, $s3->{'secret'},
+			    $s3->{'endpoint'} ]);
+		}
 	}
 foreach my $sched (grep { &can_backup_sched($_) } &list_scheduled_backups()) {
 	local @dests = &get_scheduled_backup_dests($sched);
@@ -6639,12 +6641,6 @@ sub list_all_s3_accounts
 			}
 		}
 	}
-if (&can_cloud_providers()) {
-	foreach my $s3 (&list_s3_accounts()) {
-		push(@rv, [ $s3->{'access'}, $s3->{'secret'},
-			    $s3->{'endpoint'} ]);
-		}
-	}
 local %done;
 return grep { !$done{$_->[0]}++ } @rv;
 }

cloud-lib.pl

@@ -6,38 +6,45 @@ sub list_cloud_providers
 {
 my @rv = ( { 'name' => 's3',
 	     'prefix' => [ 's3', 's3rrs' ],
+	     'clear' => 0,
 	     'url' => 'https://aws.amazon.com/s3/',
 	     'desc' => $text{'cloud_s3desc'},
 	     'longdesc' => \&cloud_s3_longdesc },
 	   { 'name' => 'rs',
 	     'prefix' => [ 'rs' ],
+	     'clear' => 0,
 	     'url' => 'https://www.rackspace.com/openstack/public/files',
 	     'desc' => $text{'cloud_rsdesc'},
 	     'longdesc' => $text{'cloud_rs_longdesc'} } );
 if ($virtualmin_pro) {
 	my $ourl = &get_miniserv_base_url()."/$module_name/oauth.cgi";
 	push(@rv, { 'name' => 'google',
 		    'prefix' => [ 'gcs' ],
+		    'clear' => 1,
 		    'url' => 'https://cloud.google.com/storage',
 		    'desc' => $text{'cloud_googledesc'},
 		    'longdesc' => \&cloud_google_longdesc });
 	push(@rv, { 'name' => 'dropbox',
 		    'prefix' => [ 'dropbox' ],
+		    'clear' => 1,
 		    'url' => 'https://www.dropbox.com/',
 		    'desc' => $text{'cloud_dropboxdesc'},
 		    'longdesc' => $text{'cloud_dropbox_longdesc'} });
 	push(@rv, { 'name' => 'bb',
 		    'prefix' => [ 'bb' ],
+		    'clear' => 1,
 		    'url' => 'https://www.backblaze.com/',
 		    'desc' => $text{'cloud_bbdesc'},
 		    'longdesc' => $text{'cloud_bb_longdesc'} });
 	push(@rv, { 'name' => 'azure',
 		    'prefix' => [ 'azure' ],
+		    'clear' => 1,
 		    'url' => 'https://azure.microsoft.com/',
 		    'desc' => $text{'cloud_azdesc'},
 		    'longdesc' => $text{'cloud_az_longdesc'} });
 	push(@rv, { 'name' => 'drive',
 		    'prefix' => [ 'drive' ],
+		    'clear' => 1,
 		    'url' => 'https://drive.google.com/',
 		    'desc' => $text{'cloud_drivedesc'},
 		    'longdesc' => \&cloud_drive_longdesc });
@@ -68,10 +75,11 @@ sub backup_uses_cloud
 
 sub cloud_s3_get_state
 {
-if ($config{'s3_akey'}) {
+my @s3s = &list_s3_accounts();
+if (@s3s) {
 	return { 'ok' => 1,
 		 'desc' => &text('cloud_s3account',
-				 "<tt>$config{'s3_akey'}</tt>"),
+				 "<tt>$s3s[0]->{'access'}</tt>"),
 	       };
 	}
 elsif (&can_use_aws_s3_creds()) {
@@ -86,36 +94,19 @@ sub cloud_s3_get_state
 
 sub cloud_s3_longdesc
 {
-if (!$config{'s3_akey'} && &can_use_aws_s3_creds()) {
+my @s3s = &list_s3_accounts();
+if (!@s3s && &can_use_aws_s3_creds()) {
 	return $text{'cloud_s3_creds'};
 	}
 else {
-	return $text{'cloud_s3_longdesc'};
+	return &text('cloud_s3_longdesc2', 'list_s3s.cgi');
 	}
 }
 
 sub cloud_s3_show_inputs
 {
 my $rv;
 
-# Default login
-if ($config{'s3_akey'} || !&can_use_aws_s3_creds()) {
-	# Prompt for login
-	$rv .= &ui_table_row($text{'cloud_s3_akey'},
-		&ui_radio("s3_akey_def", $config{'s3_akey'} ? 0 : 1,
-			  [ [ 1, $text{'cloud_noneset'} ],
-			    [ 0, $text{'cloud_below'} ] ])."<br>\n".
-		&ui_grid_table([ "<b>$text{'cloud_s3_access'}</b>",
-			 &ui_textbox("s3_akey", $config{'s3_akey'}, 50),
-			 "<b>$text{'cloud_s3_secret'}</b>",
-			 &ui_textbox("s3_skey", $config{'s3_skey'}, 50) ], 2));
-	}
-
-# S3 endpoint hostname, for non-amazon implementations
-$rv .= &ui_table_row($text{'cloud_s3_endpoint'},
-	&ui_opt_textbox("s3_endpoint", $config{'s3_endpoint'}, 40,
-			$text{'cloud_s3_amazon'}));
-
 # Upload chunk size
 $rv .= &ui_table_row($text{'cloud_s3_chunk'},
 	&ui_opt_textbox("s3_chunk", $config{'s3_chunk'}, 6,
@@ -147,33 +138,6 @@ sub cloud_s3_parse_inputs
 {
 my ($in) = @_;
 
-# Parse default login
-if ($config{'s3_akey'} || !&can_use_aws_s3_creds()) {
-	if ($in->{'s3_akey_def'}) {
-		delete($config{'s3_akey'});
-		delete($config{'s3_skey'});
-		}
-	else {
-		$in->{'s3_akey'} =~ /^\S+$/ || &error($text{'backup_eakey'});
-		$in->{'s3_skey'} =~ /^\S+$/ || &error($text{'backup_eskey'});
-		$config{'s3_akey'} = $in->{'s3_akey'};
-		$config{'s3_skey'} = $in->{'s3_skey'};
-		}
-	}
-
-# Parse endpoint hostname
-if ($in->{'s3_endpoint_def'}) {
-	delete($config{'s3_endpoint'});
-	}
-else {
-	my ($host, $port) = split(/:/, $in->{'s3_endpoint'});
-	&to_ipaddress($host) ||
-		&error($text{'cloud_es3_endpoint'});
-	!$port || $port =~ /^\d+$/ ||
-		&error($text{'cloud_es3_endport'});
-	$config{'s3_endpoint'} = $in->{'s3_endpoint'};
-	}
-
 # Parse chunk size
 if ($in->{'s3_chunk_def'}) {
 	delete($config{'s3_chunk'});
@@ -207,11 +171,7 @@ sub cloud_s3_parse_inputs
 sub cloud_s3_clear
 {
 # Clear Virtualmin's credentials
-my $akey = $config{'s3_akey'};
 &lock_file($module_config_file);
-delete($config{'s3_akey'});
-delete($config{'s3_skey'});
-delete($config{'s3_location'});
 &save_module_config();
 &unlock_file($module_config_file);
 

create-s3-bucket.pl

@@ -51,8 +51,7 @@ package virtual_server;
 		&usage("Unknown parameter $a");
 		}
 	}
-$akey ||= $config{'s3_akey'};
-$skey ||= $config{'s3_skey'};
+($akey, $skey) = &lookup_s3_credentials($akey, $skey);
 if (!&can_use_aws_s3_creds()) {
 	$akey || &usage("Missing --access-key parameter");
 	$skey || &usage("Missing --secret-key parameter");

delete-s3-bucket.pl

@@ -53,8 +53,7 @@ package virtual_server;
 		&usage("Unknown parameter $a");
 		}
 	}
-$akey ||= $config{'s3_akey'};
-$skey ||= $config{'s3_skey'};
+($akey, $skey) = &lookup_s3_credentials($akey, $skey);
 if (!&can_use_aws_s3_creds()) {
 	$akey || &usage("Missing --access-key parameter");
 	$skey || &usage("Missing --secret-key parameter");

delete-s3-file.pl

@@ -53,8 +53,7 @@ package virtual_server;
 		&usage("Unknown parameter $a");
 		}
 	}
-$akey ||= $config{'s3_akey'};
-$skey ||= $config{'s3_skey'};
+($akey, $skey) = &lookup_s3_credentials($akey, $skey);
 if (!&can_use_aws_s3_creds()) {
 	$akey || &usage("Missing --access-key parameter");
 	$skey || &usage("Missing --secret-key parameter");

download-s3-file.pl

@@ -55,8 +55,7 @@ package virtual_server;
 		&usage("Unknown parameter $a");
 		}
 	}
-$akey ||= $config{'s3_akey'};
-$skey ||= $config{'s3_skey'};
+($akey, $skey) = &lookup_s3_credentials($akey, $skey);
 if (!&can_use_aws_s3_creds()) {
 	$akey || &usage("Missing --access-key parameter");
 	$skey || &usage("Missing --secret-key parameter");

edit_cloud.cgi

@@ -83,7 +83,7 @@ print &ui_table_row($text{'cloud_users'}, $utable);
 
 print &ui_table_end();
 print &ui_form_end([ [ undef, $text{'save'} ],
-		     $state->{'ok'} ? ( [ 'clear', $text{'cloud_clear'} ] )
-				    : ( ) ]);
+		     $state->{'ok'} && $prov->{'clear'} ?
+			( [ 'clear', $text{'cloud_clear'} ] ) : ( ) ]);
 
 &ui_print_footer("list_clouds.cgi", $text{'clouds_return'});

lang/en

@@ -244,9 +244,9 @@ index_backuplog=Backup Logs
 index_backuplogdesc=Show logs of all backups that you have permissions to create, made via the Virtualmin UI, command line or on schedule.
 index_bkeys=Backup Encryption Keys
 index_bkeysdesc=List GPG keys used for signing and encrypting Virtualmin backups.
-index_buckets=Amazon S3 Buckets
+index_buckets=S3 Buckets
 index_bucketsdesc=Show S3 buckets used for backups, and allow migration to Glacier to be configured.
-index_s3s=Amazon S3 Accounts
+index_s3s=S3 Accounts
 index_s3sdesc=Show accounts at Amazon S3 and compatible providers, for use by Virtualmin backups.
 index_clouds=Cloud Storage Providers
 index_cloudsdesc=Configure authentication to cloud storage providers like S3, rackspace, Google and Dropbox.
@@ -2451,7 +2451,7 @@ backup_mode0=Local file or directory
 backup_mode0a=File under <tt>virtualmin-backup/</tt>
 backup_mode1=FTP server
 backup_mode2=SSH server
-backup_mode3=Amazon S3 bucket
+backup_mode3=S3 bucket
 backup_mode44=Download via link
 backup_mode5=Upload to server
 backup_mode6=Rackspace Cloud Files
@@ -2512,8 +2512,8 @@ backup_niceftp=$1 on FTP server $2
 backup_nicescp=$1 on SSH server $2
 backup_nicewebmin=$1 on Webmin server $2
 backup_nicefile=local file $1
-backup_nices3=Amazon S3 bucket $1
-backup_nices3p=$2 in Amazon S3 bucket $1
+backup_nices3=S3 bucket $1
+backup_nices3p=$2 in S3 bucket $1
 backup_nicers=Rackspace container $1
 backup_nicersp=$2 in Rackspace container $1
 backup_nicego=Google folder $1
@@ -2875,7 +2875,7 @@ restore_disabledomain=Re-disabling virtual server $1 ..
 restore_fordomain=Restoring backup for virtual server $1 ..
 restore_download=Downloading archive from FTP server ..
 restore_downloadssh=Downloading archive from SSH server ..
-restore_downloads3=Downloading archive from Amazon S3 server ..
+restore_downloads3=Downloading archive from S3 server ..
 restore_downloadrs=Downloading archive from Rackspace Cloud Files ..
 restore_downloadgc=Downloading archive from Google Cloud Storage ..
 restore_downloaddb=Downloading archive from Dropbox ..
@@ -7271,7 +7271,7 @@ fixmodphp_ecannot=You are not allowed to fix PHP permissions
 fixmodphp_doing=Updating all virtual servers to prevent use of mod_php when disabled ..
 fixmodphp_done=.. fixed $1 domains
 
-buckets_title=Amazon S3 Buckets
+buckets_title=S3 Buckets
 buckets_ecannot=You are not allowed to manage S3 buckets
 buckets_eaccounts=No scheduled backups to S3 have been created yet, and no default S3 keys have been set.
 buckets_none=No S3 buckets associated with any of your S3 accounts were found.
@@ -7288,7 +7288,7 @@ bucket_title2=Edit S3 Bucket
 bucket_eagone=Account does not exist!
 bucket_elist=Failed to list buckets : $1
 bucket_egone=Bucket no longer exists!
-bucket_header=Amazon S3 bucket details
+bucket_header=S3 bucket details
 bucket_account=S3 account key
 bucket_name=Bucket name
 bucket_owner=Owned by
@@ -7519,7 +7519,7 @@ cloud_google_project=Project ID
 cloud_google_location=Location for new buckets
 cloud_google_oauth=OAuth2 code from Google
 cloud_google_token=OAuth2 token from Google
-cloud_s3_longdesc=The credentials to enter on this page can be found on the Security Credentials page of your account on the <a href=https://console.aws.amazon.com/ target=_blank>AWS Console</a>.
+cloud_s3_longdesc2=The credentials for Amazon S3 or other compatible providers can be entered on the <a href='$1'>S3 Accounts</a> page.
 cloud_google_longdesc=The settings on this page are supplied by Google when you grant API access to your project on the <a href=https://console.cloud.google.com/apis/credentials target=_blank>Google Cloud Console</a> page, under <b>APIs & Services ⇾ Credentials</b> page. You must create an <b>OAuth 2.0 Client ID</b> credential, with the <b>Authorized redirect URI</b> set to <tt>$1</tt> value. Once created, the <b>client ID</b> and <b>client secret</b> can be entered here.
 cloud_dropbox_longdesc=Before you can backup to Dropbox, you must first <a href=https://www.dropbox.com/register target=_blank>create an account</a> on their website if you don't already have one.
 cloud_bb_longdesc=Before you can backup to Backblaze, you must first <a href=https://backblaze.com/ target=_blank>create an account</a>, and then enable <b>B2 Cloud Storage</b> on the <b>My Settings</b> page. Then you need to generate an Application key with all capabilities on the <b>App Keys</b> page.
@@ -7942,7 +7942,7 @@ global_dismiss=Dismiss
 ports_fpm=PHP FPM server
 ports_script=Installed $1 script
 
-s3s_title=Amazon S3 Accounts
+s3s_title=S3 Accounts
 s3s_longdesc=The S3 accounts on this page can be used by Virtualmin scheduled backups. Accounts can either be at Amazon AWS, or at other cloud storage providers that offer an S3-compatible HTTP API.
 s3s_none=No accounts have been registered yet.
 s3s_add=Add a new S3 account.
@@ -7957,7 +7957,7 @@ s3s_return=S3 accounts
 
 s3_title1=Add S3 Account
 s3_title2=Edit S3 Account
-s3_longdesc=XXX
+s3_longdesc=The credentials to enter on this page can be found on the Security Credentials page of your account on the <a href=https://console.aws.amazon.com/ target=_blank>AWS Console</a> if you are using AWS. Other S3 compatible providers will also supply an access key and secret key for your account that you can enter here, along with the provider's endpoint URL.
 s3_header=S3 account details
 s3_desc=Account description
 s3_access=S3 access key

list-s3-accounts.pl

@@ -52,7 +52,8 @@ package virtual_server;
 	# Full details
 	my @scheds = &list_scheduled_backups();
 	foreach $s (@s3s) {
-		print $s->{'access'},"\n";
+		print $s->{'id'},"\n";
+		print "    Access key: $s->{'access'}\n";
 		print "    Secret key: $s->{'secret'}\n";
 		if ($s->{'desc'}) {
 			print "    Description: $s->{'desc'}\n";

list-s3-buckets.pl

@@ -58,8 +58,7 @@ package virtual_server;
 		&usage("Unknown parameter $a");
 		}
 	}
-$akey ||= $config{'s3_akey'};
-$skey ||= $config{'s3_skey'};
+($akey, $skey) = &lookup_s3_credentials($akey, $skey);
 if (!&can_use_aws_s3_creds()) {
 	$akey || &usage("Missing --access-key parameter");
 	$skey || &usage("Missing --secret-key parameter");

list-s3-files.pl

@@ -58,8 +58,7 @@ package virtual_server;
 		&usage("Unknown parameter $a");
 		}
 	}
-$akey ||= $config{'s3_akey'};
-$skey ||= $config{'s3_skey'};
+($akey, $skey) = &lookup_s3_credentials($akey, $skey);
 if (!&can_use_aws_s3_creds()) {
 	$akey || &usage("Missing --access-key parameter");
 	$skey || &usage("Missing --secret-key parameter");

postinstall.pl

@@ -524,6 +524,9 @@ sub module_install
 		}
 	}
 
+# Create S3 account entries from scheduled backups
+&create_s3_accounts_from_backups();
+
 # Run any needed actions, like server restarts
 &run_post_actions_silently();
 

s3-lib.pl

@@ -1244,7 +1244,7 @@ sub list_s3_accounts
 		    'default' => 1, });
 	}
 if (opendir(DIR, $s3_accounts_dir)) {
-	foreach my $f (readdir(DIR)) {
+	foreach my $f (sort { $a cmp $b } readdir(DIR)) {
 		next if ($f eq "." || $f eq "..");
 		my %account;
 		&read_file("$s3_accounts_dir/$f", \%account);
@@ -1268,6 +1268,30 @@ sub get_s3_account
 return $rv;
 }
 
+# get_default_s3_account()
+# Returns the first or default S3 account
+sub get_default_s3_account
+{
+my @s3s = &list_s3_accounts();
+return undef if (!@s3s);
+my ($s3) = grep { $_->{'default'} } @s3s;
+$s3 ||= $s3s[0];
+return $s3;
+}
+
+# lookup_s3_credentials([access-key], [secret-key])
+# Returns either the default access and secret key, or the secret key from
+# the account matching the access key
+sub lookup_s3_credentials
+{
+my ($akey, $skey) = @_;
+if ($akey && $skey) {
+	return ($akey, $skey);
+	}
+my $s3 = $akey ? &get_s3_account($akey) : &get_default_s3_account();
+return $s3 ? ( $s3->{'access'}, $s3->{'secret'} ) : ( );
+}
+
 # save_s3_account(&account)
 # Create or update an S3 account
 sub save_s3_account
@@ -1344,6 +1368,7 @@ sub create_s3_accounts_from_backups
 					'secret' => $skey,
 					'desc' => "S3 account from backup ".
 						  $sched->{'desc'},
+				        'endpoint' => $config{'s3_endpoint'},
 				      };
 				&save_s3_account($s3);
 				push(@s3s, $s3);

upload-s3-file.pl

@@ -78,8 +78,7 @@ package virtual_server;
 		&usage("Unknown parameter $a");
 		}
 	}
-$akey ||= $config{'s3_akey'};
-$skey ||= $config{'s3_skey'};
+($akey, $skey) = &lookup_s3_credentials($akey, $skey);
 if (!&can_use_aws_s3_creds()) {
 	$akey || &usage("Missing --access-key parameter");
 	$skey || &usage("Missing --secret-key parameter");