plugin: lvmplugin: add underlay functions

src/PVE/Storage.pm

@@ -15,7 +15,7 @@ use Socket;
 use Time::Local qw(timelocal);
 
 use PVE::Tools qw(run_command file_read_firstline dir_glob_foreach $IPV6RE);
-use PVE::Cluster qw(cfs_read_file cfs_write_file cfs_lock_file);
+use PVE::Cluster qw(cfs_read_file cfs_write_file cfs_lock_file cfs_register_file);
 use PVE::DataCenterConfig;
 use PVE::Exception qw(raise_param_exc raise);
 use PVE::JSONSchema;
@@ -239,6 +239,76 @@ sub write_config {
     cfs_write_file('storage.cfg', $cfg);
 }
 
+cfs_register_file("extend-queue", \&parser_extend_queue, \&writer_extend_queue);
+
+sub extend_queue {
+    return cfs_read_file("extend-queue");
+}
+
+sub write_extend_queue {
+    my ($extend_queue) = @_;
+    return cfs_write_file("extend-queue",$extend_queue);
+}
+
+sub lock_extend_queue {
+    my ($code, $errmsg) = @_;
+
+    cfs_lock_file("extend-queue", undef, $code);
+    my $err = $@;
+    if ($err) {
+        $errmsg ? die "$errmsg: $err" : die $err;
+    }
+}
+
+sub parser_extend_queue {
+    my ($filename, $raw) = @_;
+
+    my @queue;
+
+    my $lineno = 0;
+    my @lines = split(/\n/, $raw);
+    my $nextline = sub {
+        while (defined(my $line = shift @lines)) {
+            $lineno++;
+            return $line if ($line !~ /^\s*#/);
+        }
+    };
+
+    while (@lines) {
+        my $line = $nextline->();
+        next if !$line;
+        print "Current line $line\n";
+
+        # vmid: nodename
+        if ($line =~ '[1-9][0-9]{2,8}+: [aefz][0-9a-f]{30}') {
+            print "Extend request is valid\n";
+            my ($vmid, $nodename) = split(/:\s/, $line, 2);
+            push @queue, [$vmid, $nodename];
+        }
+    }
+    return \@queue;
+}
+
+sub writer_extend_queue {
+    my ($filename, $queue) = @_;
+
+    my $out = "";
+    foreach my $entry (@$queue) {
+        my ($vmid, $nodename) = @$entry;
+        $out .= format_extend_request($vmid, $nodename) . "\n";
+    }
+
+    return $out;
+}
+
+sub format_extend_request {
+    my ($vmid, $node_name) = @_;
+
+    my $request = $vmid . ': ' . $node_name;
+
+    return $request;
+}
+
 sub lock_storage_config {
     my ($code, $errmsg) = @_;
 
@@ -410,6 +480,34 @@ sub volume_resize {
     }
 }
 
+sub volume_underlay_size_info {
+    my ($cfg, $volid, $timeout) = @_;
+
+    my ($storeid, $volname) = parse_volume_id($volid, 1);
+    if ($storeid) {
+        my $scfg = storage_config($cfg, $storeid);
+        my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
+        return $plugin->volume_underlay_size_info($scfg, $storeid, $volname, $timeout);
+    } else {
+        return 0;
+    }
+}
+
+sub volume_underlay_resize {
+    my ($cfg, $volid, $size, $running, $backing_snap) = @_;
+
+    my ($storeid, $volname) = parse_volume_id($volid, 1);
+    if ($storeid) {
+        my $scfg = storage_config($cfg, $storeid);
+        my $plugin = PVE::Storage::Plugin->lookup($scfg->{type});
+        return $plugin->volume_underlay_resize($scfg, $storeid, $volname, $size, $running, $backing_snap);
+    } elsif ($volid =~ m|^(/.+)$| && -e $volid) {
+        die "resize file/device '$volid' is not possible\n";
+    } else {
+        die "unable to parse volume ID '$volid'\n";
+    }
+}
+
 sub volume_rollback_is_possible {
     my ($cfg, $volid, $snap, $blockers) = @_;
 

src/PVE/Storage/Common.pm

@@ -170,7 +170,7 @@ C<$options> currently allows setting the C<preallocation> value.
 =cut
 
 sub qemu_img_create_qcow2_backed {
-    my ($path, $backing_path, $backing_format, $options) = @_;
+    my ($path, $backing_path, $backing_format, $options, $thin) = @_;
 
     my $cmd = [
         '/usr/bin/qemu-img',
@@ -188,7 +188,7 @@ sub qemu_img_create_qcow2_backed {
     my $opts = ['extended_l2=on', 'cluster_size=128k'];
 
     push @$opts, "preallocation=$options->{preallocation}"
-        if defined($options->{preallocation});
+        if defined($options->{preallocation}) && !$thin;
     push @$cmd, '-o', join(',', @$opts) if @$opts > 0;
 
     run_command($cmd, errmsg => "unable to create image");

src/PVE/Storage/LVMPlugin.pm

@@ -400,6 +400,8 @@ sub options {
         tagged_only => { optional => 1 },
         bwlimit => { optional => 1 },
         'snapshot-as-volume-chain' => { optional => 1 },
+        chunksize => { optional => 1 },
+        'chunk-percentage' => { optional => 1 },
     };
 }
 
@@ -575,7 +577,7 @@ sub lvrename {
 }
 
 my sub lvm_qcow2_format {
-    my ($class, $storeid, $scfg, $name, $fmt, $backing_snap, $size) = @_;
+    my ($class, $storeid, $scfg, $name, $fmt, $backing_snap, $size, $thin) = @_;
 
     $class->activate_volume($storeid, $scfg, $name);
     my $path = $class->path($scfg, $name, $storeid);
@@ -585,7 +587,9 @@ my sub lvm_qcow2_format {
     };
     if ($backing_snap) {
         my $backing_volname = get_snap_name($class, $name, $backing_snap);
-        PVE::Storage::Common::qemu_img_create_qcow2_backed($path, $backing_volname, $fmt, $options);
+        PVE::Storage::Common::qemu_img_create_qcow2_backed(
+            $path, $backing_volname, $fmt, $options, $thin,
+        );
     } else {
         PVE::Storage::Common::qemu_img_create($fmt, $size, $path, $options);
     }
@@ -629,7 +633,16 @@ my sub alloc_lvm_image {
     die "no such volume group '$vg'\n" if !defined($vgs->{$vg});
 
     my $free = int($vgs->{$vg}->{free});
-    my $lvmsize = calculate_lvm_size($size, $fmt, $backing_snap);
+    my $lvmsize;
+
+    # FIX: make this variable a check box when taking a snapshot
+    # right now all snapshots are created thin for testing purposes
+    my $thin = $backing_snap ? 1 : 0;
+    if ($thin) {
+        $lvmsize = 2 * 1024 * 1024;
+    } else {
+        $lvmsize = calculate_lvm_size($size, $fmt, $backing_snap);
+    }
 
     die "not enough free space ($free < $size)\n" if $free < $size;
 
@@ -641,7 +654,7 @@ my sub alloc_lvm_image {
     return if $fmt ne 'qcow2';
 
     #format the lvm volume with qcow2 format
-    eval { lvm_qcow2_format($class, $storeid, $scfg, $name, $fmt, $backing_snap, $size) };
+    eval { lvm_qcow2_format($class, $storeid, $scfg, $name, $fmt, $backing_snap, $size, $thin) };
     if ($@) {
         my $err = $@;
         #no need to safe cleanup as the volume is still empty
@@ -928,6 +941,44 @@ sub volume_resize {
     $lvmsize = "${lvmsize}k";
 
     my $path = $class->path($scfg, $volname);
+    lv_extend($class, $scfg, $storeid, $lvmsize, $path);
+
+    if (!$running && $format eq 'qcow2') {
+        my $preallocation = PVE::Storage::Plugin::preallocation_cmd_opt($scfg, $format);
+        PVE::Storage::Common::qemu_img_resize($path, $format, $size, $preallocation, 10);
+    }
+
+    return 1;
+}
+
+sub volume_underlay_resize {
+    my ($class, $scfg, $storeid, $volname, $backing_snap) = @_;
+
+    my ($format) = ($class->parse_volname($volname))[6];
+
+    my $path = $class->filesystem_path($scfg, $volname);
+    my $json = PVE::Storage::Common::qemu_img_info($path, undef, 10, 0);
+    my $json_decode = eval { decode_json($json) };
+    if ($@) {
+        die "Can't decode qemu snapshot list. Invalid JSON: $@\n";
+    }
+
+    my $virtual_size = $json_decode->{'virtual-size'} / 1024;
+
+    my $underlay_size = lv_size($path, 10);
+
+    my $updated_underlay_size = ($underlay_size + $scfg->{chunksize}) / 1024;
+    $updated_underlay_size = calculate_lvm_size($virtual_size, $format, $backing_snap)
+        if $updated_underlay_size >= $virtual_size;
+
+    my $lvmsize = "${updated_underlay_size}k";
+    lv_extend($class, $scfg, $storeid, $lvmsize, $path);
+
+    return $updated_underlay_size;
+}
+
+sub lv_extend {
+    my ($class, $scfg, $storeid, $lvmsize, $path) = @_;
     my $cmd = ['/sbin/lvextend', '-L', $lvmsize, $path];
 
     $class->cluster_lock_storage(
@@ -938,13 +989,6 @@ sub volume_resize {
             run_command($cmd, errmsg => "error resizing volume '$path'");
         },
     );
-
-    if (!$running && $format eq 'qcow2') {
-        my $preallocation = PVE::Storage::Plugin::preallocation_cmd_opt($scfg, $format);
-        PVE::Storage::Common::qemu_img_resize($path, $format, $size, $preallocation, 10);
-    }
-
-    return 1;
 }
 
 sub volume_size_info {
@@ -955,6 +999,22 @@ sub volume_size_info {
 
     return PVE::Storage::Plugin::file_size_info($path, $timeout, $format) if $format eq 'qcow2';
 
+    my $size = lv_size($path, $timeout);
+    return wantarray ? ($size, 'raw', 0, undef) : $size;
+}
+
+sub volume_underlay_size_info {
+    my ($class, $scfg, $storeid, $volname, $timeout) = @_;
+
+    my ($format) = ($class->parse_volname($volname))[6];
+    my $path = $class->filesystem_path($scfg, $volname);
+
+    return lv_size($path, $timeout);
+}
+
+sub lv_size {
+    my ($path, $timeout) = @_;
+
     my $cmd = [
         '/sbin/lvs',
         '--separator',
@@ -978,7 +1038,7 @@ sub volume_size_info {
             $size = int(shift);
         },
     );
-    return wantarray ? ($size, 'raw', 0, undef) : $size;
+    return $size;
 }
 
 sub volume_snapshot {

src/PVE/Storage/Plugin.pm

@@ -228,6 +228,20 @@ my $defaultData = {
             default => 0,
             optional => 1,
         },
+        chunksize => {
+            type => 'integer',
+            description => 'The chunksize in Bytes to define the write threshold'
+                . 'of thin disks on thick storage.',
+            default => 1073741824, # 1 GiB
+            optional => 1,
+        },
+        'chunk-percentage' => {
+            type => 'number',
+            description => 'The percentage of written disk to define the write'
+                . 'threshold.',
+            default => 0.5,
+            optional => 1,
+        },
     },
 };
 
@@ -1265,7 +1279,6 @@ sub volume_size_info {
     my $format = ($class->parse_volname($volname))[6];
     my $path = $class->filesystem_path($scfg, $volname);
     return file_size_info($path, $timeout, $format);
-
 }
 
 sub volume_resize {
@@ -1285,6 +1298,20 @@ sub volume_resize {
     return undef;
 }
 
+sub volume_underlay_size_info {
+    my ($class, $scfg, $storeid, $volname, $timeout) = @_;
+
+    # Only supported by LVM for now
+    die "volume underlay is not supported for storage type '$scfg->{type}'\n";
+}
+
+sub volume_underlay_resize {
+    my ($class, $scfg, $storeid, $volname, $backing_snap) = @_;
+
+    # Only supported by LVM for now
+    die "volume underlay is not supported for storage type '$scfg->{type}'\n";
+}
+
 sub volume_snapshot {
     my ($class, $scfg, $storeid, $volname, $snap) = @_;