[win_get_url] feature: Add support checksum to module win_get_url (#51986)

* set valid_until equal to current time + spot_wait_timeout

* Add checksum check for downloaded file.

* refactoring

* fix typo

* add fixes

* mart try,catch handling

* revert lib/ansible/modules/cloud/amazon/ec2.py from upstream

* refactoring

* remove empty lines

* add checksum verification for existing file

* fix current file check

* refactoring destination file check

* add handling exceptions

* refactoring

* Added download file hash data from url

* fix string aligning

* fix bug with uri

* Added get hash from multy-string file

* Added URI support for checksum file location

* refactoing

* Remove any non-alphanumeric characters for hash from url

* fix discussions; add support for PS3

* refactoring

* add size return value

* checkout from upstream for lib/ansible/modules/cloud/amazon/ec2.py

* add Ansible.ModuleUtils.Legacy support; refactoring

* Copyright added

* Checking files size before and after downloading added.

* remove unused code

* Corrected regexp for dotted slashed file name prefix in hash-file

* hotfix typo error; add int tests

* remove legacy module support; split checksum to checksum, checksum_algorithm, checksum_url

* changed default hash algorithm

* Fixed case for ContentLength = -1

* Old comment removed

* fix typo

* Remove file size check before downloading

* add alias to ; fix tests

* adjust tests; fix lint warnings from PSScritpAnalyzer

* workaround for bug in win_chocolatey module on win2008

* remove win_get_url.ps1 from /test/sanity/pslint/ignore.txt

* add checksum_algorithm as retuen value

* first normalise before return Result

* resolve discussions

Signed-off-by: Viktor Utkin <viktor.utkin7@yandex.ru>

* fix discussions
fix http tests as discussed

* fix last discussions

* Reduce code duplication and add idempotency check

* fix sanity issue and remove testing code

* move back to using tmp file for checksum comparison

test/integration/targets/win_get_url/defaults/main.yml

@@ -2,3 +2,6 @@
 test_win_get_url_path: '{{win_output_dir}}\win_get_url'
 test_win_get_url_host: www.redhat.com
 test_win_get_url_env_var: WIN_GET_URL
+
+remote_tmp_path: '{{win_output_dir}}\win_get_url_dest'
+remote_http_path: '{{test_win_get_url_path}}\http'

test/integration/targets/win_get_url/tasks/main.yml

@@ -1,8 +1,12 @@
 ---
 - name: ensure testing folder is present
   win_file:
-    path: '{{test_win_get_url_path}}'
+    path: '{{ item }}'
     state: directory
+  loop:
+    - '{{ test_win_get_url_path }}'
+    - '{{ remote_http_path }}'
+    - '{{ remote_tmp_path }}'
 
 - name: copy across testing files
   win_copy:
@@ -41,6 +45,9 @@
   - name: run FTP tests
     include_tasks: tests_ftp.yml
 
+  - name: run checksum tests
+    include_tasks: tests_checksum.yml
+
   always:
   - name: remove SlimFTPd service
     win_service:
@@ -53,7 +60,10 @@
       level: machine
       state: absent
 
-  - name: remove testing folder
+  - name: remove testing folders
     win_file:
-      path: '{{test_win_get_url_path}}'
+      path: '{{ item }}'
       state: absent
+    loop:
+      - '{{ test_win_get_url_path }}'
+      - '{{ remote_tmp_path }}'

test/integration/targets/win_get_url/tasks/tests_checksum.yml

@@ -0,0 +1,202 @@
+---
+- name: create src file
+  win_copy:
+    dest: '{{ remote_http_path }}\27617.txt'
+    content: 'ptux'
+
+- name: create sha1 checksum file of src
+  win_copy:
+    dest: '{{ remote_http_path }}\sha1sum.txt'
+    content: 'a97e6837f60cec6da4491bab387296bbcd72bdba  27617.txt'
+
+- name: add sha1 checksum not going to be downloaded
+  win_lineinfile:
+    dest: '{{ remote_http_path }}\sha1sum.txt'
+    line: '{{ item }}'
+  loop:
+    - '3911340502960ca33aece01129234460bfeb2791  not_target1.txt'
+    - '1b4b6adf30992cedb0f6edefd6478ff0a593b2e4  not_target2.txt'
+
+- name: create sha256 checksum file of src
+  win_copy:
+    dest: '{{ remote_http_path }}\sha256sum.txt'
+    content: 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006.  27617.txt'
+
+- name: add sha256 checksum not going to be downloaded
+  win_lineinfile:
+    dest: '{{ remote_http_path }}\sha256sum.txt'
+    line: '{{ item }}'
+  loop:
+    - '30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892  not_target1.txt'
+    - 'd0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b  not_target2.txt'
+
+- name: create sha256 checksum file of src with a dot leading path
+  win_copy:
+    dest: '{{ remote_http_path }}\sha256sum_with_dot.txt'
+    content: 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006.  ./27617.txt'
+
+- name: add sha256 checksums with dot leading path not going to be downloaded
+  win_lineinfile:
+    dest: '{{ remote_http_path }}\sha256sum_with_dot.txt'
+    line: '{{ item }}'
+  loop:
+    - '30949cc401e30ac494d695ab8764a9f76aae17c5d73c67f65e9b558f47eff892  ./not_target1.txt'
+    - 'd0dbfc1945bc83bf6606b770e442035f2c4e15c886ee0c22fb3901ba19900b5b  ./not_target2.txt'
+
+- name: copy files to ftp
+  win_copy:
+    src: '{{ remote_http_path }}\'
+    dest: '{{ test_win_get_url_path }}\ftp\{{ item }}\'
+    remote_src: yes
+  loop: '{{ dest_folders }}'
+  vars:
+    dest_folders:
+      - anon
+      - user
+      - user-pass
+
+- name: download src with sha1 checksum url
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum_url: 'ftp://localhost/anon/sha1sum.txt'
+    force: True
+  register: result_sha1
+
+- name: download src with sha1 checksum value
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum: 'a97e6837f60cec6da4491bab387296bbcd72bdba'
+    force: True
+  register: result_sha1_alt
+
+- win_stat:
+    path: '{{ remote_tmp_path }}\27617.txt'
+  register: stat_result_sha1
+
+- name: download src with sha256 checksum url
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}\27617sha256.txt'
+    checksum_url: 'ftp://localhost/anon/sha256sum.txt'
+    checksum_algorithm: sha256
+    force: True
+  register: result_sha256
+
+- win_stat:
+    path: '{{ remote_tmp_path }}\27617.txt'
+  register: stat_result_sha256
+
+- name: download src with sha256 checksum url with dot leading paths
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}\27617sha256_with_dot.txt'
+    checksum_url: 'ftp://localhost/anon/sha256sum_with_dot.txt'
+    checksum_algorithm: sha256
+    force: True
+  register: result_sha256_with_dot
+
+- win_stat:
+    path: '{{ remote_tmp_path }}\27617sha256_with_dot.txt'
+  register: stat_result_sha256_with_dot
+
+- name: Assert that the file was downloaded
+  assert:
+    that:
+      - result_sha1 is changed
+      - result_sha1.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba'
+      - result_sha1_alt is succeeded
+      - not result_sha1_alt is changed
+      - result_sha1_alt.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba'
+      - result_sha256 is changed
+      - result_sha256_with_dot is changed
+      - stat_result_sha1.stat.exists | bool
+      - stat_result_sha256.stat.exists | bool
+      - stat_result_sha256_with_dot.stat.exists | bool
+
+# Check download with force: False
+
+- name: download src with sha1 checksum url
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum_url: 'ftp://localhost/anon/sha1sum.txt'
+    checksum_algorithm: sha1
+    force: False
+  register: result_sha1_no_force
+
+- name: download src with sha256 checksum url
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}/27617sha256.txt'
+    checksum_url: 'ftp://localhost/anon/sha256sum.txt'
+    checksum_algorithm: sha256
+    force: False
+  register: result_sha256_no_force
+
+- name: assert download single file with force no
+  assert:
+    that:
+      - result_sha1_no_force is not changed
+      - result_sha256_no_force is not changed
+
+# Check download with check_mode: True
+
+- name: download src with sha1 checksum url | checkmode
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum_url: 'ftp://localhost/anon/sha1sum.txt'
+    checksum_algorithm: sha1
+  check_mode: True
+  register: result_sha1_check_mode
+
+- name: download src with sha256 checksum url | checkmode
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}\27617sha256.txt'
+    checksum_url: 'ftp://localhost/anon/sha256sum.txt'
+    checksum_algorithm: sha256
+  check_mode: True
+  register: result_sha256_check_mode
+
+- name: assert download single file with force no
+  assert:
+    that:
+      - result_sha1_check_mode.checksum_dest == 'a97e6837f60cec6da4491bab387296bbcd72bdba'
+      - result_sha256_check_mode.checksum_dest == 'b1b6ce5073c8fac263a8fc5edfffdbd5dec1980c784e09c5bc69f8fb6056f006'
+      - result_sha1_check_mode.checksum_src | length
+      - result_sha256_check_mode.checksum_src | length
+
+# Check download with wrong checksum
+
+- name: download src with sha1 checksum value | fail
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}'
+    checksum: 'redacted'
+    checksum_algorithm: sha1
+  failed_when:
+    - '"did not match" not in result_sha1_failed.msg'
+    - '"The checksum" not in result_sha1_failed.msg'
+    - '"Unknown error downloading" not in result_sha1_failed.msg'
+  register: result_sha1_failed
+
+- name: download src with sha256 checksum value | fail
+  win_get_url:
+    url: 'ftp://localhost/anon/27617.txt'
+    dest: '{{ remote_tmp_path }}\27617sha256.txt'
+    checksum: 'redacted'
+    checksum_algorithm: sha256
+  failed_when:
+    - '"did not match" not in result_sha256_failed.msg'
+    - '"The checksum" not in result_sha256_failed.msg'
+    - '"Unknown error downloading" not in result_sha256_failed.msg'
+  register: result_sha256_failed
+
+- name: assert failed downloads
+  assert:
+    that:
+      - result_sha1_failed is succeeded
+      - result_sha256_failed is succeeded

test/integration/targets/win_get_url/tasks/tests_url.yml

@@ -39,7 +39,16 @@
     - http_download.dest
     - http_download_result.stat.exists
 
-# TODO: add check for idempotent run once it is added with force: yes
+- name: download single file (idempotent)
+  win_get_url:
+    url: https://{{test_win_get_url_host}}
+    dest: '{{test_win_get_url_path}}\web.html'
+  register: http_download_again
+
+- name: assert download single file (idempotent)
+  assert:
+    that:
+    - not http_download_again is changed
 
 - name: download single file with force no
   win_get_url:
@@ -53,8 +62,10 @@
     that:
     - http_download_no_force is not changed
 
-- name: manually change last modified time on FTP source to older datetime
-  win_shell: (Get-Item -Path '{{test_win_get_url_path}}\web.html').LastWriteTime = (Get-Date -Date "01/01/1970")
+- name: manually change the content and last modified time on FTP source to older datetime
+  win_shell: |
+    Set-Content -Path '{{test_win_get_url_path}}\web.html' -Value 'abc'
+    (Get-Item -Path '{{test_win_get_url_path}}\web.html').LastWriteTime = (Get-Date -Date "01/01/1970")
 
 - name: download newer file with force no
   win_get_url: