From 01ba7f730b9af8a81f88a5b12fb8a7031e624829 Mon Sep 17 00:00:00 2001 From: Kenny Woodson Date: Tue, 19 Jan 2016 14:30:06 -0500 Subject: Fixing yaml validation in python. Inputs behave differently as does glob --- git/yaml_validate.py | 63 ----------------------------------------------- git/yaml_validation.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 63 deletions(-) delete mode 100755 git/yaml_validate.py create mode 100755 git/yaml_validation.py (limited to 'git') diff --git a/git/yaml_validate.py b/git/yaml_validate.py deleted file mode 100755 index 7e0a08a4b..000000000 --- a/git/yaml_validate.py +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env python -# -# python yaml validator for a git commit -# -''' -python yaml validator for a git commit -''' -import shutil -import sys -import os -import glob -import tempfile -import subprocess -import yaml - -def get_changes(oldrev, newrev, tempdir): - '''Get a list of git changes from oldrev to newrev''' - proc = subprocess.Popen(['/usr/bin/git', 'diff', '--name-only', oldrev, - newrev, '--diff-filter=ACM'], stdout=subprocess.PIPE) - proc.wait() - files = proc.stdout.read().strip().split('\n') - - # No file changes - if not files: - return [] - - cmd = '/usr/bin/git archive %s %s | /bin/tar x -C %s' % (newrev, " ".join(files), tempdir) - proc = subprocess.Popen(cmd, shell=True) - proc.wait() - - return [fmod for fmod in glob.glob('%s/**/*' % tempdir) if not os.path.isdir(fmod)] - -def main(): - ''' - Perform yaml validation - ''' - results = [] - try: - tmpdir = tempfile.mkdtemp(prefix='jenkins-git-') - old, new, _ = sys.argv[1:] - - for file_mod in get_changes(old, new, tmpdir): - - print "+++++++ Received: %s" % file_mod - - if not file_mod.endswith('.yml') or not file_mod.endswith('.yaml'): - continue - - try: - yaml.load(file_mod) - results.append(True) - - except yaml.scanner.ScannerError as yerr: - print yerr.message - results.append(False) - finally: - shutil.rmtree(tmpdir) - - if not all(results): - sys.exit(1) - -if __name__ == "__main__": - main() diff --git a/git/yaml_validation.py b/git/yaml_validation.py new file mode 100755 index 000000000..aa909b584 --- /dev/null +++ b/git/yaml_validation.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# +# python yaml validator for a git commit +# +''' +python yaml validator for a git commit +''' +import shutil +import sys +import os +import tempfile +import subprocess +import yaml + +def get_changes(oldrev, newrev, tempdir): + '''Get a list of git changes from oldrev to newrev''' + proc = subprocess.Popen(['/usr/bin/git', 'diff', '--name-only', oldrev, + newrev, '--diff-filter=ACM'], stdout=subprocess.PIPE) + stdout, _ = proc.communicate() + files = stdout.split('\n') + + # No file changes + if not files: + return [] + + cmd = '/usr/bin/git archive %s %s | /bin/tar x -C %s' % (newrev, " ".join(files), tempdir) + proc = subprocess.Popen(cmd, shell=True) + _, _ = proc.communicate() + + rfiles = [] + for dirpath, _, fnames in os.walk(tempdir): + for fname in fnames: + rfiles.append(os.path.join(dirpath, fname)) + + return rfiles + +def main(): + ''' + Perform yaml validation + ''' + results = [] + try: + tmpdir = tempfile.mkdtemp(prefix='jenkins-git-') + old, new, _ = sys.argv[1:] + + for file_mod in get_changes(old, new, tmpdir): + + print "+++++++ Received: %s" % file_mod + + if not file_mod.endswith('.yml') or not file_mod.endswith('.yaml'): + continue + + try: + yaml.load(file_mod) + results.append(True) + + except yaml.scanner.ScannerError as yerr: + print yerr.message + results.append(False) + finally: + shutil.rmtree(tmpdir) + + if not all(results): + sys.exit(1) + +if __name__ == "__main__": + main() -- cgit v1.2.3 From 952858ca62acf92879662339353ab819f8e0e30d Mon Sep 17 00:00:00 2001 From: Kenny Woodson Date: Tue, 19 Jan 2016 14:53:02 -0500 Subject: Fixed a logic bug and yaml load --- git/yaml_validation.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'git') diff --git a/git/yaml_validation.py b/git/yaml_validation.py index aa909b584..94b8b0435 100755 --- a/git/yaml_validation.py +++ b/git/yaml_validation.py @@ -47,15 +47,15 @@ def main(): print "+++++++ Received: %s" % file_mod - if not file_mod.endswith('.yml') or not file_mod.endswith('.yaml'): + if not file_mod.endswith('.yml') and not file_mod.endswith('.yaml'): continue try: - yaml.load(file_mod) + yaml.load(open(file_mod)) results.append(True) except yaml.scanner.ScannerError as yerr: - print yerr.message + print yerr results.append(False) finally: shutil.rmtree(tmpdir) @@ -65,3 +65,4 @@ def main(): if __name__ == "__main__": main() + -- cgit v1.2.3 From 3f9c84336fb3b1cfdfb0872af1cd6ed7c5daa2b2 Mon Sep 17 00:00:00 2001 From: Kenny Woodson Date: Tue, 19 Jan 2016 15:08:52 -0500 Subject: Removing ruby scripts and replacing with python. --- git/parent.rb | 45 ------------------------------- git/yaml_validation.rb | 72 -------------------------------------------------- 2 files changed, 117 deletions(-) delete mode 100755 git/parent.rb delete mode 100755 git/yaml_validation.rb (limited to 'git') diff --git a/git/parent.rb b/git/parent.rb deleted file mode 100755 index 2acb127c4..000000000 --- a/git/parent.rb +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env ruby -# -# -# - -if __FILE__ == $0 - # If we aren't on master we don't need to parent check - branch = 'prod' - exit(0) if ARGV[0] !~ /#{branch}/ - commit_id = ARGV[1] - %x[/usr/bin/git checkout #{branch}] - %x[/usr/bin/git merge #{commit_id}] - - count = 0 - #lines = %x[/usr/bin/git rev-list --left-right stg...master].split("\n") - lines = %x[/usr/bin/git rev-list --left-right remotes/origin/stg...#{branch}].split("\n") - lines.each do |commit| - # next if they are in stage - next if commit =~ /^' - commit = commit[1..-1] - # check if any remote branches contain $commit - results = %x[/usr/bin/git branch -q -r --contains #{commit} 2>/dev/null ] - # if this comes back empty, nothing contains it, we can skip it as - # we have probably created the merge commit here locally - next if results.empty? - - # The results generally contain origin/pr/246/merge and origin/pr/246/head - # this is the pull request which would contain the commit in question. - # - # If the results do not contain origin/stg then stage does not contain - # the commit in question. Therefore we need to alert! - unless results =~ /origin\/stg/ - puts "\nFAILED: (These commits are not in stage.)\n" - puts "\t#{commit}" - count += 1 - end - end - - # Exit with count of commits in #{branch} but not stg - exit(count) -end - -__END__ - diff --git a/git/yaml_validation.rb b/git/yaml_validation.rb deleted file mode 100755 index f5ded7a78..000000000 --- a/git/yaml_validation.rb +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env ruby -# -# -# -require 'yaml' -require 'tmpdir' - -class YamlValidate - def self.yaml_file?(filename) - return filename.end_with?('.yaml') || filename.end_with?('.yml') - end - - def self.short_yaml_ext?(filename) - return filename.end_with?(".yml") - end - - def self.valid_yaml?(filename) - YAML::load_file(filename) - - return true - end -end - -class GitCommit - attr_accessor :oldrev, :newrev, :refname, :tmp - def initialize(oldrev, newrev, refname) - @oldrev = oldrev - @newrev = newrev - @refname = refname - @tmp = Dir.mktmpdir(@newrev) - end - - def get_file_changes() - files = %x[/usr/bin/git diff --name-only #{@oldrev} #{@newrev} --diff-filter=ACM].split("\n") - - # if files is empty we will get a full checkout. This happens on - # a git rm file. If there are no changes then we need to skip the archive - return [] if files.empty? - - # We only want to take the files that changed. Archive will do that when passed - # the filenames. It will export these to a tmp dir - system("/usr/bin/git archive #{@newrev} #{files.join(" ")} | tar x -C #{@tmp}") - return Dir.glob("#{@tmp}/**/*").delete_if { |file| File.directory?(file) } - end -end - -if __FILE__ == $0 - while data = STDIN.gets - oldrev, newrev, refname = data.split - gc = GitCommit.new(oldrev, newrev, refname) - - results = [] - gc.get_file_changes().each do |file| - begin - puts "++++++ Received: #{file}" - - #raise "Yaml file extensions must be .yaml not .yml" if YamlValidate.short_yaml_ext? file - - # skip readme, other files, etc - next unless YamlValidate.yaml_file?(file) - - results << YamlValidate.valid_yaml?(file) - rescue Exception => ex - puts "\n#{ex.message}\n\n" - results << false - end - end - - #puts "RESULTS\n#{results.inspect}\n" - exit 1 if results.include?(false) - end -end -- cgit v1.2.3