diff options
Diffstat (limited to 'qemu/scripts')
46 files changed, 4044 insertions, 3254 deletions
diff --git a/qemu/scripts/acpi_extract.py b/qemu/scripts/acpi_extract.py deleted file mode 100755 index 10c1ffb36..000000000 --- a/qemu/scripts/acpi_extract.py +++ /dev/null @@ -1,367 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2011 Red Hat, Inc., Michael S. Tsirkin <mst@redhat.com> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -# Process mixed ASL/AML listing (.lst file) produced by iasl -l -# Locate and execute ACPI_EXTRACT directives, output offset info -# -# Documentation of ACPI_EXTRACT_* directive tags: -# -# These directive tags output offset information from AML for BIOS runtime -# table generation. -# Each directive is of the form: -# ACPI_EXTRACT_<TYPE> <array_name> <Operator> (...) -# and causes the extractor to create an array -# named <array_name> with offset, in the generated AML, -# of an object of a given type in the following <Operator>. -# -# A directive must fit on a single code line. -# -# Object type in AML is verified, a mismatch causes a build failure. -# -# Directives and operators currently supported are: -# ACPI_EXTRACT_NAME_DWORD_CONST - extract a Dword Const object from Name() -# ACPI_EXTRACT_NAME_WORD_CONST - extract a Word Const object from Name() -# ACPI_EXTRACT_NAME_BYTE_CONST - extract a Byte Const object from Name() -# ACPI_EXTRACT_METHOD_STRING - extract a NameString from Method() -# ACPI_EXTRACT_NAME_STRING - extract a NameString from Name() -# ACPI_EXTRACT_PROCESSOR_START - start of Processor() block -# ACPI_EXTRACT_PROCESSOR_STRING - extract a NameString from Processor() -# ACPI_EXTRACT_PROCESSOR_END - offset at last byte of Processor() + 1 -# ACPI_EXTRACT_PKG_START - start of Package block -# -# ACPI_EXTRACT_ALL_CODE - create an array storing the generated AML bytecode -# -# ACPI_EXTRACT is not allowed anywhere else in code, except in comments. - -import re; -import sys; -import fileinput; - -aml = [] -asl = [] -output = {} -debug = "" - -class asl_line: - line = None - lineno = None - aml_offset = None - -def die(diag): - sys.stderr.write("Error: %s; %s\n" % (diag, debug)) - sys.exit(1) - -#Store an ASL command, matching AML offset, and input line (for debugging) -def add_asl(lineno, line): - l = asl_line() - l.line = line - l.lineno = lineno - l.aml_offset = len(aml) - asl.append(l) - -#Store an AML byte sequence -#Verify that offset output by iasl matches # of bytes so far -def add_aml(offset, line): - o = int(offset, 16); - # Sanity check: offset must match size of code so far - if (o != len(aml)): - die("Offset 0x%x != 0x%x" % (o, len(aml))) - # Strip any trailing dots and ASCII dump after " - line = re.sub(r'\s*\.*\s*".*$',"", line) - # Strip traling whitespace - line = re.sub(r'\s+$',"", line) - # Strip leading whitespace - line = re.sub(r'^\s+',"", line) - # Split on whitespace - code = re.split(r'\s+', line) - for c in code: - # Require a legal hex number, two digits - if (not(re.search(r'^[0-9A-Fa-f][0-9A-Fa-f]$', c))): - die("Unexpected octet %s" % c); - aml.append(int(c, 16)); - -# Process aml bytecode array, decoding AML -def aml_pkglen_bytes(offset): - # PkgLength can be multibyte. Bits 8-7 give the # of extra bytes. - pkglenbytes = aml[offset] >> 6; - return pkglenbytes + 1 - -def aml_pkglen(offset): - pkgstart = offset - pkglenbytes = aml_pkglen_bytes(offset) - pkglen = aml[offset] & 0x3F - # If multibyte, first nibble only uses bits 0-3 - if ((pkglenbytes > 1) and (pkglen & 0x30)): - die("PkgLen bytes 0x%x but first nibble 0x%x expected 0x0X" % - (pkglen, pkglen)) - offset += 1 - pkglenbytes -= 1 - for i in range(pkglenbytes): - pkglen |= aml[offset + i] << (i * 8 + 4) - if (len(aml) < pkgstart + pkglen): - die("PckgLen 0x%x at offset 0x%x exceeds AML size 0x%x" % - (pkglen, offset, len(aml))) - return pkglen - -# Given method offset, find its NameString offset -def aml_method_string(offset): - #0x14 MethodOp PkgLength NameString MethodFlags TermList - if (aml[offset] != 0x14): - die( "Method offset 0x%x: expected 0x14 actual 0x%x" % - (offset, aml[offset])); - offset += 1; - pkglenbytes = aml_pkglen_bytes(offset) - offset += pkglenbytes; - return offset; - -# Given name offset, find its NameString offset -def aml_name_string(offset): - #0x08 NameOp NameString DataRef - if (aml[offset] != 0x08): - die( "Name offset 0x%x: expected 0x08 actual 0x%x" % - (offset, aml[offset])); - offset += 1 - # Block Name Modifier. Skip it. - if (aml[offset] == 0x5c or aml[offset] == 0x5e): - offset += 1 - return offset; - -# Given data offset, find variable length byte buffer offset -def aml_data_buffer(offset, length): - #0x11 PkgLength BufferSize ByteList - if (length > 63): - die( "Name offset 0x%x: expected a one byte PkgLength (length<=63)" % - (offset)); - expect = [0x11, length+3, 0x0A, length] - if (aml[offset:offset+4] != expect): - die( "Name offset 0x%x: expected %s actual %s" % - (offset, expect, aml[offset:offset+4])) - return offset + len(expect) - -# Given data offset, find dword const offset -def aml_data_dword_const(offset): - #0x08 NameOp NameString DataRef - if (aml[offset] != 0x0C): - die( "Name offset 0x%x: expected 0x0C actual 0x%x" % - (offset, aml[offset])); - return offset + 1; - -# Given data offset, find word const offset -def aml_data_word_const(offset): - #0x08 NameOp NameString DataRef - if (aml[offset] != 0x0B): - die( "Name offset 0x%x: expected 0x0B actual 0x%x" % - (offset, aml[offset])); - return offset + 1; - -# Given data offset, find byte const offset -def aml_data_byte_const(offset): - #0x08 NameOp NameString DataRef - if (aml[offset] != 0x0A): - die( "Name offset 0x%x: expected 0x0A actual 0x%x" % - (offset, aml[offset])); - return offset + 1; - -# Find name'd buffer -def aml_name_buffer(offset, length): - return aml_data_buffer(aml_name_string(offset) + 4, length) - -# Given name offset, find dword const offset -def aml_name_dword_const(offset): - return aml_data_dword_const(aml_name_string(offset) + 4) - -# Given name offset, find word const offset -def aml_name_word_const(offset): - return aml_data_word_const(aml_name_string(offset) + 4) - -# Given name offset, find byte const offset -def aml_name_byte_const(offset): - return aml_data_byte_const(aml_name_string(offset) + 4) - -def aml_device_start(offset): - #0x5B 0x82 DeviceOp PkgLength NameString - if ((aml[offset] != 0x5B) or (aml[offset + 1] != 0x82)): - die( "Name offset 0x%x: expected 0x5B 0x82 actual 0x%x 0x%x" % - (offset, aml[offset], aml[offset + 1])); - return offset - -def aml_device_string(offset): - #0x5B 0x82 DeviceOp PkgLength NameString - start = aml_device_start(offset) - offset += 2 - pkglenbytes = aml_pkglen_bytes(offset) - offset += pkglenbytes - return offset - -def aml_device_end(offset): - start = aml_device_start(offset) - offset += 2 - pkglenbytes = aml_pkglen_bytes(offset) - pkglen = aml_pkglen(offset) - return offset + pkglen - -def aml_processor_start(offset): - #0x5B 0x83 ProcessorOp PkgLength NameString ProcID - if ((aml[offset] != 0x5B) or (aml[offset + 1] != 0x83)): - die( "Name offset 0x%x: expected 0x5B 0x83 actual 0x%x 0x%x" % - (offset, aml[offset], aml[offset + 1])); - return offset - -def aml_processor_string(offset): - #0x5B 0x83 ProcessorOp PkgLength NameString ProcID - start = aml_processor_start(offset) - offset += 2 - pkglenbytes = aml_pkglen_bytes(offset) - offset += pkglenbytes - return offset - -def aml_processor_end(offset): - start = aml_processor_start(offset) - offset += 2 - pkglenbytes = aml_pkglen_bytes(offset) - pkglen = aml_pkglen(offset) - return offset + pkglen - -def aml_package_start(offset): - offset = aml_name_string(offset) + 4 - # 0x12 PkgLength NumElements PackageElementList - if (aml[offset] != 0x12): - die( "Name offset 0x%x: expected 0x12 actual 0x%x" % - (offset, aml[offset])); - offset += 1 - return offset + aml_pkglen_bytes(offset) + 1 - -lineno = 0 -for line in fileinput.input(): - # Strip trailing newline - line = line.rstrip(); - # line number and debug string to output in case of errors - lineno = lineno + 1 - debug = "input line %d: %s" % (lineno, line) - #ASL listing: space, then line#, then ...., then code - pasl = re.compile('^\s+([0-9]+)(:\s\s|\.\.\.\.)\s*') - m = pasl.search(line) - if (m): - add_asl(lineno, pasl.sub("", line)); - # AML listing: offset in hex, then ...., then code - paml = re.compile('^([0-9A-Fa-f]+)(:\s\s|\.\.\.\.)\s*') - m = paml.search(line) - if (m): - add_aml(m.group(1), paml.sub("", line)) - -# Now go over code -# Track AML offset of a previous non-empty ASL command -prev_aml_offset = -1 -for i in range(len(asl)): - debug = "input line %d: %s" % (asl[i].lineno, asl[i].line) - - l = asl[i].line - - # skip if not an extract directive - a = len(re.findall(r'ACPI_EXTRACT', l)) - if (not a): - # If not empty, store AML offset. Will be used for sanity checks - # IASL seems to put {}. at random places in the listing. - # Ignore any non-words for the purpose of this test. - m = re.search(r'\w+', l) - if (m): - prev_aml_offset = asl[i].aml_offset - continue - - if (a > 1): - die("Expected at most one ACPI_EXTRACT per line, actual %d" % a) - - mext = re.search(r''' - ^\s* # leading whitespace - /\*\s* # start C comment - (ACPI_EXTRACT_\w+) # directive: group(1) - \s+ # whitspace separates directive from array name - (\w+) # array name: group(2) - \s*\*/ # end of C comment - \s*$ # trailing whitespace - ''', l, re.VERBOSE) - if (not mext): - die("Stray ACPI_EXTRACT in input") - - # previous command must have produced some AML, - # otherwise we are in a middle of a block - if (prev_aml_offset == asl[i].aml_offset): - die("ACPI_EXTRACT directive in the middle of a block") - - directive = mext.group(1) - array = mext.group(2) - offset = asl[i].aml_offset - - if (directive == "ACPI_EXTRACT_ALL_CODE"): - if array in output: - die("%s directive used more than once" % directive) - output[array] = aml - continue - if (directive == "ACPI_EXTRACT_NAME_BUFFER8"): - offset = aml_name_buffer(offset, 8) - elif (directive == "ACPI_EXTRACT_NAME_BUFFER16"): - offset = aml_name_buffer(offset, 16) - elif (directive == "ACPI_EXTRACT_NAME_DWORD_CONST"): - offset = aml_name_dword_const(offset) - elif (directive == "ACPI_EXTRACT_NAME_WORD_CONST"): - offset = aml_name_word_const(offset) - elif (directive == "ACPI_EXTRACT_NAME_BYTE_CONST"): - offset = aml_name_byte_const(offset) - elif (directive == "ACPI_EXTRACT_NAME_STRING"): - offset = aml_name_string(offset) - elif (directive == "ACPI_EXTRACT_METHOD_STRING"): - offset = aml_method_string(offset) - elif (directive == "ACPI_EXTRACT_DEVICE_START"): - offset = aml_device_start(offset) - elif (directive == "ACPI_EXTRACT_DEVICE_STRING"): - offset = aml_device_string(offset) - elif (directive == "ACPI_EXTRACT_DEVICE_END"): - offset = aml_device_end(offset) - elif (directive == "ACPI_EXTRACT_PROCESSOR_START"): - offset = aml_processor_start(offset) - elif (directive == "ACPI_EXTRACT_PROCESSOR_STRING"): - offset = aml_processor_string(offset) - elif (directive == "ACPI_EXTRACT_PROCESSOR_END"): - offset = aml_processor_end(offset) - elif (directive == "ACPI_EXTRACT_PKG_START"): - offset = aml_package_start(offset) - else: - die("Unsupported directive %s" % directive) - - if array not in output: - output[array] = [] - output[array].append(offset) - -debug = "at end of file" - -def get_value_type(maxvalue): - #Use type large enough to fit the table - if (maxvalue >= 0x10000): - return "int" - elif (maxvalue >= 0x100): - return "short" - else: - return "char" - -# Pretty print output -for array in output.keys(): - otype = get_value_type(max(output[array])) - odata = [] - for value in output[array]: - odata.append("0x%x" % value) - sys.stdout.write("static unsigned %s %s[] = {\n" % (otype, array)) - sys.stdout.write(",\n".join(odata)) - sys.stdout.write('\n};\n'); diff --git a/qemu/scripts/acpi_extract_preprocess.py b/qemu/scripts/acpi_extract_preprocess.py deleted file mode 100755 index 69d10d621..000000000 --- a/qemu/scripts/acpi_extract_preprocess.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2011 Red Hat, Inc., Michael S. Tsirkin <mst@redhat.com> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see <http://www.gnu.org/licenses/>. - -# Read a preprocessed ASL listing and put each ACPI_EXTRACT -# directive in a comment, to make iasl skip it. -# We also put each directive on a new line, the machinery -# in tools/acpi_extract.py requires this. - -import re; -import sys; -import fileinput; - -def die(diag): - sys.stderr.write("Error: %s\n" % (diag)) - sys.exit(1) - -# Note: () around pattern make split return matched string as part of list -psplit = re.compile(r''' ( - \b # At word boundary - ACPI_EXTRACT_\w+ # directive - \s+ # some whitespace - \w+ # array name - )''', re.VERBOSE); - -lineno = 0 -for line in fileinput.input(): - # line number and debug string to output in case of errors - lineno = lineno + 1 - debug = "input line %d: %s" % (lineno, line.rstrip()) - - s = psplit.split(line); - # The way split works, each odd item is the matching ACPI_EXTRACT directive. - # Put each in a comment, and on a line by itself. - for i in range(len(s)): - if (i % 2): - sys.stdout.write("\n/* %s */\n" % s[i]) - else: - sys.stdout.write(s[i]) diff --git a/qemu/scripts/analyze-migration.py b/qemu/scripts/analyze-migration.py index f6894bece..14553876a 100755 --- a/qemu/scripts/analyze-migration.py +++ b/qemu/scripts/analyze-migration.py @@ -252,6 +252,15 @@ class HTABSection(object): def getDict(self): return "" + +class ConfigurationSection(object): + def __init__(self, file): + self.file = file + + def read(self): + name_len = self.file.read32() + name = self.file.readstr(len = name_len) + class VMSDFieldGeneric(object): def __init__(self, desc, file): self.file = file @@ -474,6 +483,7 @@ class MigrationDump(object): QEMU_VM_SECTION_FULL = 0x04 QEMU_VM_SUBSECTION = 0x05 QEMU_VM_VMDESCRIPTION = 0x06 + QEMU_VM_CONFIGURATION = 0x07 QEMU_VM_SECTION_FOOTER= 0x7e def __init__(self, filename): @@ -514,6 +524,9 @@ class MigrationDump(object): section_type = file.read8() if section_type == self.QEMU_VM_EOF: break + elif section_type == self.QEMU_VM_CONFIGURATION: + section = ConfigurationSection(file) + section.read() elif section_type == self.QEMU_VM_SECTION_START or section_type == self.QEMU_VM_SECTION_FULL: section_id = file.read32() name = file.readstr() diff --git a/qemu/scripts/checkpatch.pl b/qemu/scripts/checkpatch.pl index 7f0aae977..c9554ba64 100755 --- a/qemu/scripts/checkpatch.pl +++ b/qemu/scripts/checkpatch.pl @@ -141,44 +141,22 @@ our $Ident = qr{ }x; our $Storage = qr{extern|static|asmlinkage}; our $Sparse = qr{ - __user| - __kernel| - __force| - __iomem| - __must_check| - __init_refok| - __kprobes| - __ref + __force }x; # Notes to $Attribute: -# We need \b after 'init' otherwise 'initconst' will cause a false positive in a check our $Attribute = qr{ const| - __percpu| - __nocast| - __safe| - __bitwise__| - __packed__| - __packed2__| - __naked| - __maybe_unused| - __always_unused| - __noreturn| - __used| - __cold| - __noclone| - __deprecated| - __read_mostly| - __kprobes| - __(?:mem|cpu|dev|)(?:initdata|initconst|init\b)| - ____cacheline_aligned| - ____cacheline_aligned_in_smp| - ____cacheline_internodealigned_in_smp| - __weak + volatile| + QEMU_NORETURN| + QEMU_WARN_UNUSED_RESULT| + QEMU_SENTINEL| + QEMU_ARTIFICIAL| + QEMU_PACKED| + GCC_FMT_ATTR }x; our $Modifier; -our $Inline = qr{inline|__always_inline|noinline}; +our $Inline = qr{inline}; our $Member = qr{->$Ident|\.$Ident|\[[^]]*\]}; our $Lval = qr{$Ident(?:$Member)*}; @@ -215,14 +193,6 @@ our $typeTypedefs = qr{(?x: | QEMUBH # all uppercase )}; -our $logFunctions = qr{(?x: - printk| - pr_(debug|dbg|vdbg|devel|info|warning|err|notice|alert|crit|emerg|cont)| - (dev|netdev|netif)_(printk|dbg|vdbg|info|warn|err|notice|alert|crit|emerg|WARN)| - WARN| - panic -)}; - our @typeList = ( qr{void}, qr{(?:unsigned\s+)?char}, @@ -242,21 +212,22 @@ our @typeList = ( qr{${Ident}_t}, qr{${Ident}_handler}, qr{${Ident}_handler_fn}, + qr{target_(?:u)?long}, ); + +# This can be modified by sub possible. Since it can be empty, be careful +# about regexes that always match, because they can cause infinite loops. our @modifierList = ( - qr{fastcall}, ); -our $allowed_asm_includes = qr{(?x: - irq| - memory -)}; -# memory.h: ARM has a custom one - sub build_types { - my $mods = "(?x: \n" . join("|\n ", @modifierList) . "\n)"; my $all = "(?x: \n" . join("|\n ", @typeList) . "\n)"; - $Modifier = qr{(?:$Attribute|$Sparse|$mods)}; + if (@modifierList > 0) { + my $mods = "(?x: \n" . join("|\n ", @modifierList) . "\n)"; + $Modifier = qr{(?:$Attribute|$Sparse|$mods)}; + } else { + $Modifier = qr{(?:$Attribute|$Sparse)}; + } $NonptrType = qr{ (?:$Modifier\s+|const\s+)* (?: @@ -277,27 +248,6 @@ build_types(); $chk_signoff = 0 if ($file); -my @dep_includes = (); -my @dep_functions = (); -my $removal = "Documentation/feature-removal-schedule.txt"; -if ($tree && -f "$root/$removal") { - open(my $REMOVE, '<', "$root/$removal") || - die "$P: $removal: open failed - $!\n"; - while (<$REMOVE>) { - if (/^Check:\s+(.*\S)/) { - for my $entry (split(/[, ]+/, $1)) { - if ($entry =~ m@include/(.*)@) { - push(@dep_includes, $1); - - } elsif ($entry !~ m@/@) { - push(@dep_functions, $entry); - } - } - } - } - close($REMOVE); -} - my @rawlines = (); my @lines = (); my $vname; @@ -633,7 +583,7 @@ sub statement_block_size { my ($stmt) = @_; $stmt =~ s/(^|\n)./$1/g; - $stmt =~ s/^\s*{//; + $stmt =~ s/^\s*\{//; $stmt =~ s/}\s*$//; $stmt =~ s/^\s*//; $stmt =~ s/\s*$//; @@ -1061,7 +1011,9 @@ sub possible { case| else| asm|__asm__| - do + do| + \#| + \#\# )(?:\s|$)| ^(?:typedef|struct|enum)\b )}x; @@ -1127,33 +1079,6 @@ sub CHK { } } -sub check_absolute_file { - my ($absolute, $herecurr) = @_; - my $file = $absolute; - - ##print "absolute<$absolute>\n"; - - # See if any suffix of this path is a path within the tree. - while ($file =~ s@^[^/]*/@@) { - if (-f "$root/$file") { - ##print "file<$file>\n"; - last; - } - } - if (! -f _) { - return 0; - } - - # It is, so see if the prefix is acceptable. - my $prefix = $absolute; - substr($prefix, -length($file)) = ''; - - ##print "prefix<$prefix>\n"; - if ($prefix ne ".../") { - WARN("use relative pathname instead of absolute in changelog text\n" . $herecurr); - } -} - sub process { my $filename = shift; @@ -1196,10 +1121,6 @@ sub process { my %suppress_export; # Pre-scan the patch sanitizing the lines. - # Pre-scan the patch looking for any __setup documentation. - # - my @setup_docs = (); - my $setup_docs = 0; sanitise_line_reset(); my $line; @@ -1207,13 +1128,6 @@ sub process { $linenr++; $line = $rawline; - if ($rawline=~/^\+\+\+\s+(\S+)/) { - $setup_docs = 0; - if ($1 =~ m@Documentation/kernel-parameters.txt$@) { - $setup_docs = 1; - } - #next; - } if ($rawline=~/^\@\@ -\d+(?:,\d+)? \+(\d+)(,(\d+))? \@\@/) { $realline=$1-1; if (defined $2) { @@ -1272,10 +1186,6 @@ sub process { #print "==>$rawline\n"; #print "-->$line\n"; - - if ($setup_docs && $line =~ /^\+/) { - push(@setup_docs, $line); - } } $prefix = ''; @@ -1350,9 +1260,6 @@ sub process { WARN("patch prefix '$p1_prefix' exists, appears to be a -p0 patch\n"); } - if ($realfile =~ m@^include/asm/@) { - ERROR("do not modify files in include/asm, change architecture specific files in include/asm-<architecture>\n" . "$here$rawline\n"); - } next; } @@ -1367,7 +1274,7 @@ sub process { # Check for incorrect file permissions if ($line =~ /^new (file )?mode.*[7531]\d{0,2}$/) { my $permhere = $here . "FILE: $realfile\n"; - if ($realfile =~ /(Makefile|Kconfig|\.c|\.cpp|\.h|\.S|\.tmpl)$/) { + if ($realfile =~ /(\bMakefile(?:\.objs)?|\.c|\.cc|\.cpp|\.h|\.mak|\.[sS])$/) { ERROR("do not set execute permissions for source files\n" . $permhere); } } @@ -1392,20 +1299,6 @@ sub process { $herecurr) if (!$emitted_corrupt++); } -# Check for absolute kernel paths. - if ($tree) { - while ($line =~ m{(?:^|\s)(/\S*)}g) { - my $file = $1; - - if ($file =~ m{^(.*?)(?::\d+)+:?$} && - check_absolute_file($1, $herecurr)) { - # - } else { - check_absolute_file($file, $herecurr); - } - } - } - # UTF-8 regex found at http://www.w3.org/International/questions/qa-forms-utf-8.en.php if (($realfile =~ /^$/ || $line =~ /^\+/) && $rawline !~ m/^$UTF8*$/) { @@ -1432,45 +1325,12 @@ sub process { $rpt_cleaners = 1; } -# check for Kconfig help text having a real description -# Only applies when adding the entry originally, after that we do not have -# sufficient context to determine whether it is indeed long enough. - if ($realfile =~ /Kconfig/ && - $line =~ /\+\s*(?:---)?help(?:---)?$/) { - my $length = 0; - my $cnt = $realcnt; - my $ln = $linenr + 1; - my $f; - my $is_end = 0; - while ($cnt > 0 && defined $lines[$ln - 1]) { - $f = $lines[$ln - 1]; - $cnt-- if ($lines[$ln - 1] !~ /^-/); - $is_end = $lines[$ln - 1] =~ /^\+/; - $ln++; - - next if ($f =~ /^-/); - $f =~ s/^.//; - $f =~ s/#.*//; - $f =~ s/^\s+//; - next if ($f =~ /^$/); - if ($f =~ /^\s*config\s/) { - $is_end = 1; - last; - } - $length++; - } - WARN("please write a paragraph that describes the config symbol fully\n" . $herecurr) if ($is_end && $length < 4); - #print "is_end<$is_end> length<$length>\n"; - } - # check we are in a valid source file if not then ignore this hunk next if ($realfile !~ /\.(h|c|cpp|s|S|pl|sh)$/); #80 column limit - if ($line =~ /^\+/ && $prevrawline !~ /\/\*\*/ && - $rawline !~ /^.\s*\*\s*\@$Ident\s/ && - !($line =~ /^\+\s*$logFunctions\s*\(\s*(?:(KERN_\S+\s*|[^"]*))?"[X\t]*"\s*(?:,|\)\s*;)\s*$/ || - $line =~ /^\+\s*"[^"]*"\s*(?:\s*|,|\)\s*;)\s*$/) && + if ($line =~ /^\+/ && + !($line =~ /^\+\s*"[^"]*"\s*(?:\s*|,|\)\s*;)\s*$/) && $length > 80) { WARN("line over 80 characters\n" . $herecurr); @@ -1486,18 +1346,6 @@ sub process { WARN("adding a line without newline at end of file\n" . $herecurr); } -# Blackfin: use hi/lo macros - if ($realfile =~ m@arch/blackfin/.*\.S$@) { - if ($line =~ /\.[lL][[:space:]]*=.*&[[:space:]]*0x[fF][fF][fF][fF]/) { - my $herevet = "$here\n" . cat_vet($line) . "\n"; - ERROR("use the LO() macro, not (... & 0xFFFF)\n" . $herevet); - } - if ($line =~ /\.[hH][[:space:]]*=.*>>[[:space:]]*16/) { - my $herevet = "$here\n" . cat_vet($line) . "\n"; - ERROR("use the HI() macro, not (... >> 16)\n" . $herevet); - } - } - # check we are in a valid source file C or perl if not then ignore this hunk next if ($realfile !~ /\.(h|c|cpp|pl)$/); @@ -1516,16 +1364,6 @@ sub process { WARN("CVS style keyword markers, these will _not_ be updated\n". $herecurr); } -# Blackfin: don't use __builtin_bfin_[cs]sync - if ($line =~ /__builtin_bfin_csync/) { - my $herevet = "$here\n" . cat_vet($line) . "\n"; - ERROR("use the CSYNC() macro in asm/blackfin.h\n" . $herevet); - } - if ($line =~ /__builtin_bfin_ssync/) { - my $herevet = "$here\n" . cat_vet($line) . "\n"; - ERROR("use the SSYNC() macro in asm/blackfin.h\n" . $herevet); - } - # Check for potential 'bare' types my ($stat, $cond, $line_nr_next, $remain_next, $off_next, $realline_next); @@ -1644,7 +1482,7 @@ sub process { # 79 or 80 characters, it is no longer possible to add a space and an # opening brace there) if ($#ctx == 0 && $ctx !~ /{\s*/ && - defined($lines[$ctx_ln - 1]) && $lines[$ctx_ln - 1] =~ /^\+\s*{/ && + defined($lines[$ctx_ln - 1]) && $lines[$ctx_ln - 1] =~ /^\+\s*\{/ && defined($lines[$ctx_ln - 2]) && length($lines[$ctx_ln - 2]) < 80) { ERROR("that open brace { should be on the previous line\n" . "$here\n$ctx\n$rawlines[$ctx_ln - 1]\n"); @@ -1684,7 +1522,7 @@ sub process { my $continuation = 0; my $check = 0; $s =~ s/^.*\bdo\b//; - $s =~ s/^\s*{//; + $s =~ s/^\s*\{//; if ($s =~ s/^\s*\\//) { $continuation = 1; } @@ -1783,7 +1621,7 @@ sub process { } # check for initialisation to aggregates open brace on the next line - if ($line =~ /^.\s*{/ && + if ($line =~ /^.\s*\{/ && $prevline =~ /(?:^|[^=])=\s*$/) { ERROR("that open brace { should be on the previous line\n" . $hereprev); } @@ -1809,50 +1647,6 @@ sub process { $line =~ s@//.*@@; $opline =~ s@//.*@@; -# EXPORT_SYMBOL should immediately follow the thing it is exporting, consider -# the whole statement. -#print "APW <$lines[$realline_next - 1]>\n"; - if (defined $realline_next && - exists $lines[$realline_next - 1] && - !defined $suppress_export{$realline_next} && - ($lines[$realline_next - 1] =~ /EXPORT_SYMBOL.*\((.*)\)/ || - $lines[$realline_next - 1] =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) { - # Handle definitions which produce identifiers with - # a prefix: - # XXX(foo); - # EXPORT_SYMBOL(something_foo); - my $name = $1; - if ($stat =~ /^.([A-Z_]+)\s*\(\s*($Ident)/ && - $name =~ /^${Ident}_$2/) { -#print "FOO C name<$name>\n"; - $suppress_export{$realline_next} = 1; - - } elsif ($stat !~ /(?: - \n.}\s*$| - ^.DEFINE_$Ident\(\Q$name\E\)| - ^.DECLARE_$Ident\(\Q$name\E\)| - ^.LIST_HEAD\(\Q$name\E\)| - ^.(?:$Storage\s+)?$Type\s*\(\s*\*\s*\Q$name\E\s*\)\s*\(| - \b\Q$name\E(?:\s+$Attribute)*\s*(?:;|=|\[|\() - )/x) { -#print "FOO A<$lines[$realline_next - 1]> stat<$stat> name<$name>\n"; - $suppress_export{$realline_next} = 2; - } else { - $suppress_export{$realline_next} = 1; - } - } - if (!defined $suppress_export{$linenr} && - $prevline =~ /^.\s*$/ && - ($line =~ /EXPORT_SYMBOL.*\((.*)\)/ || - $line =~ /EXPORT_UNUSED_SYMBOL.*\((.*)\)/)) { -#print "FOO B <$lines[$linenr - 1]>\n"; - $suppress_export{$linenr} = 2; - } - if (defined $suppress_export{$linenr} && - $suppress_export{$linenr} == 2) { - WARN("EXPORT_SYMBOL(foo); should immediately follow its function/variable\n" . $herecurr); - } - # check for global initialisers. if ($line =~ /^.$Type\s*$Ident\s*(?:\s+$Modifier)*\s*=\s*(0|NULL|false)\s*;/) { ERROR("do not initialise globals to 0 or NULL\n" . @@ -1900,67 +1694,37 @@ sub process { } } -# # no BUG() or BUG_ON() -# if ($line =~ /\b(BUG|BUG_ON)\b/) { -# print "Try to use WARN_ON & Recovery code rather than BUG() or BUG_ON()\n"; -# print "$herecurr"; -# $clean = 0; -# } - - if ($line =~ /\bLINUX_VERSION_CODE\b/) { - WARN("LINUX_VERSION_CODE should be avoided, code should be for the version to which it is merged\n" . $herecurr); - } - -# printk should use KERN_* levels. Note that follow on printk's on the -# same line do not need a level, so we use the current block context -# to try and find and validate the current printk. In summary the current -# printk includes all preceding printk's which have no newline on the end. -# we assume the first bad printk is the one to report. - if ($line =~ /\bprintk\((?!KERN_)\s*"/) { - my $ok = 0; - for (my $ln = $linenr - 1; $ln >= $first_line; $ln--) { - #print "CHECK<$lines[$ln - 1]\n"; - # we have a preceding printk if it ends - # with "\n" ignore it, else it is to blame - if ($lines[$ln - 1] =~ m{\bprintk\(}) { - if ($rawlines[$ln - 1] !~ m{\\n"}) { - $ok = 1; - } - last; - } - } - if ($ok == 0) { - WARN("printk() should include KERN_ facility level\n" . $herecurr); - } - } - # function brace can't be on same line, except for #defines of do while, # or if closed on same line - if (($line=~/$Type\s*$Ident\(.*\).*\s{/) and - !($line=~/\#\s*define.*do\s{/) and !($line=~/}/)) { + if (($line=~/$Type\s*$Ident\(.*\).*\s\{/) and + !($line=~/\#\s*define.*do\s\{/) and !($line=~/}/)) { ERROR("open brace '{' following function declarations go on the next line\n" . $herecurr); } # open braces for enum, union and struct go on the same line. - if ($line =~ /^.\s*{/ && + if ($line =~ /^.\s*\{/ && $prevline =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?\s*$/) { ERROR("open brace '{' following $1 go on the same line\n" . $hereprev); } # missing space after union, struct or enum definition if ($line =~ /^.\s*(?:typedef\s+)?(enum|union|struct)(?:\s+$Ident)?(?:\s+$Ident)?[=\{]/) { - WARN("missing space after $1 definition\n" . $herecurr); + ERROR("missing space after $1 definition\n" . $herecurr); } # check for spacing round square brackets; allowed: # 1. with a type on the left -- int [] a; # 2. at the beginning of a line for slice initialisers -- [0...10] = 5, # 3. inside a curly brace -- = { [0...10] = 5 } +# 4. after a comma -- [1] = 5, [2] = 6 +# 5. in a macro definition -- #define abc(x) [x] = y while ($line =~ /(.*?\s)\[/g) { my ($where, $prefix) = ($-[1], $1); if ($prefix !~ /$Type\s+$/ && ($where != 0 || $prefix !~ /^.\s+$/) && - $prefix !~ /{\s+$/) { + $prefix !~ /{\s+$/ && + $prefix !~ /\#\s*define[^(]*\([^)]*\)\s+$/ && + $prefix !~ /,\s+$/) { ERROR("space prohibited before open square bracket '['\n" . $herecurr); } } @@ -2091,7 +1855,7 @@ sub process { # not required when having a single },{ on one line } elsif ($op eq ',') { if ($ctx !~ /.x[WEC]/ && $cc !~ /^}/ && - ($elements[$n] . $elements[$n + 2]) !~ " *}{") { + ($elements[$n] . $elements[$n + 2]) !~ " *}\\{") { ERROR("space required after that '$op' $at\n" . $hereptr); } @@ -2131,19 +1895,6 @@ sub process { ERROR("space prohibited after that '$op' $at\n" . $hereptr); } - - # << and >> may either have or not have spaces both sides - } elsif ($op eq '<<' or $op eq '>>' or - $op eq '&' or $op eq '^' or $op eq '|' or - $op eq '+' or $op eq '-' or - $op eq '*' or $op eq '/' or - $op eq '%') - { - if ($ctx =~ /Wx[^WCE]|[^WCE]xW/) { - ERROR("need consistent spacing around '$op' $at\n" . - $hereptr); - } - # A colon needs no spaces before when it is # terminating a case value or a label. } elsif ($opv eq ':C' || $opv eq ':L') { @@ -2190,29 +1941,9 @@ sub process { } } -# check for multiple assignments - if ($line =~ /^.\s*$Lval\s*=\s*$Lval\s*=(?!=)/) { - CHK("multiple assignments should be avoided\n" . $herecurr); - } - -## # check for multiple declarations, allowing for a function declaration -## # continuation. -## if ($line =~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Ident.*/ && -## $line !~ /^.\s*$Type\s+$Ident(?:\s*=[^,{]*)?\s*,\s*$Type\s*$Ident.*/) { -## -## # Remove any bracketed sections to ensure we do not -## # falsly report the parameters of functions. -## my $ln = $line; -## while ($ln =~ s/\([^\(\)]*\)//g) { -## } -## if ($ln =~ /,/) { -## WARN("declaring multiple variables together should be avoided\n" . $herecurr); -## } -## } - #need space before brace following if, while, etc - if (($line =~ /\(.*\){/ && $line !~ /\($Type\){/) || - $line =~ /do{/) { + if (($line =~ /\(.*\)\{/ && $line !~ /\($Type\)\{/) || + $line =~ /do\{/) { ERROR("space required before the open brace '{'\n" . $herecurr); } @@ -2267,7 +1998,7 @@ sub process { if ($line =~ /^.\s*return\s*(E[A-Z]*)\s*;/) { my $name = $1; if ($name ne 'EOF' && $name ne 'ERROR') { - CHK("return of an errno should typically be -ve (return -$1)\n" . $herecurr); + WARN("return of an errno should typically be -ve (return -$1)\n" . $herecurr); } } @@ -2398,22 +2129,6 @@ sub process { WARN("Whitepspace after \\ makes next lines useless\n" . $herecurr); } -#warn if <asm/foo.h> is #included and <linux/foo.h> is available (uses RAW line) - if ($tree && $rawline =~ m{^.\s*\#\s*include\s*\<asm\/(.*)\.h\>}) { - my $file = "$1.h"; - my $checkfile = "include/linux/$file"; - if (-f "$root/$checkfile" && - $realfile ne $checkfile && - $1 !~ /$allowed_asm_includes/) - { - if ($realfile =~ m{^arch/}) { - CHK("Consider using #include <linux/$file> instead of <asm/$file>\n" . $herecurr); - } else { - WARN("Use #include <linux/$file> instead of <asm/$file>\n" . $herecurr); - } - } - } - # multi-statement macros should be enclosed in a do while loop, grab the # first statement and ensure its the whole macro if its not enclosed # in a known good container @@ -2508,15 +2223,6 @@ sub process { } } -# make sure symbols are always wrapped with VMLINUX_SYMBOL() ... -# all assignments may have only one of the following with an assignment: -# . -# ALIGN(...) -# VMLINUX_SYMBOL(...) - if ($realfile eq 'vmlinux.lds.h' && $line =~ /(?:(?:^|\s)$Ident\s*=|=\s*$Ident(?:\s|$))/) { - WARN("vmlinux.lds.h needs VMLINUX_SYMBOL() around C-visible symbols\n" . $herecurr); - } - # check for missing bracing round if etc if ($line =~ /(^.*)\bif\b/ && $line !~ /\#\s*if/) { my ($level, $endln, @chunks) = @@ -2551,7 +2257,7 @@ sub process { my $spaced_block = $block; $spaced_block =~ s/\n\+/ /g; - $seen++ if ($spaced_block =~ /^\s*{/); + $seen++ if ($spaced_block =~ /^\s*\{/); print "APW: cond<$cond> block<$block> allowed<$allowed>\n" if $dbg_adv_apw; @@ -2644,64 +2350,23 @@ sub process { } } -# don't include deprecated include files (uses RAW line) - for my $inc (@dep_includes) { - if ($rawline =~ m@^.\s*\#\s*include\s*\<$inc>@) { - ERROR("Don't use <$inc>: see Documentation/feature-removal-schedule.txt\n" . $herecurr); - } - } - -# don't use deprecated functions - for my $func (@dep_functions) { - if ($line =~ /\b$func\b/) { - ERROR("Don't use $func(): see Documentation/feature-removal-schedule.txt\n" . $herecurr); - } - } - # no volatiles please my $asm_volatile = qr{\b(__asm__|asm)\s+(__volatile__|volatile)\b}; if ($line =~ /\bvolatile\b/ && $line !~ /$asm_volatile/) { WARN("Use of volatile is usually wrong: see Documentation/volatile-considered-harmful.txt\n" . $herecurr); } -# SPIN_LOCK_UNLOCKED & RW_LOCK_UNLOCKED are deprecated - if ($line =~ /\b(SPIN_LOCK_UNLOCKED|RW_LOCK_UNLOCKED)/) { - ERROR("Use of $1 is deprecated: see Documentation/spinlocks.txt\n" . $herecurr); - } - # warn about #if 0 if ($line =~ /^.\s*\#\s*if\s+0\b/) { - CHK("if this code is redundant consider removing it\n" . + WARN("if this code is redundant consider removing it\n" . $herecurr); } -# check for needless kfree() checks +# check for needless g_free() checks if ($prevline =~ /\bif\s*\(([^\)]*)\)/) { my $expr = $1; - if ($line =~ /\bkfree\(\Q$expr\E\);/) { - WARN("kfree(NULL) is safe this check is probably not required\n" . $hereprev); - } - } -# check for needless usb_free_urb() checks - if ($prevline =~ /\bif\s*\(([^\)]*)\)/) { - my $expr = $1; - if ($line =~ /\busb_free_urb\(\Q$expr\E\);/) { - WARN("usb_free_urb(NULL) is safe this check is probably not required\n" . $hereprev); - } - } - -# prefer usleep_range over udelay - if ($line =~ /\budelay\s*\(\s*(\w+)\s*\)/) { - # ignore udelay's < 10, however - if (! (($1 =~ /(\d+)/) && ($1 < 10)) ) { - CHK("usleep_range is preferred over udelay; see Documentation/timers/timers-howto.txt\n" . $line); - } - } - -# warn about unexpectedly long msleep's - if ($line =~ /\bmsleep\s*\((\d+)\);/) { - if ($1 < 20) { - WARN("msleep < 20ms can sleep for up to 20ms; see Documentation/timers/timers-howto.txt\n" . $line); + if ($line =~ /\bg_free\(\Q$expr\E\);/) { + WARN("g_free(NULL) is safe this check is probably not required\n" . $hereprev); } } @@ -2716,24 +2381,17 @@ sub process { if ($line =~ /^.\s*\#\s*(ifdef|ifndef|elif)\s\s+/) { ERROR("exactly one space required after that #$1\n" . $herecurr); } - -# check for spinlock_t definitions without a comment. - if ($line =~ /^.\s*(struct\s+mutex|spinlock_t)\s+\S+;/ || - $line =~ /^.\s*(DEFINE_MUTEX)\s*\(/) { - my $which = $1; - if (!ctx_has_comment($first_line, $linenr)) { - CHK("$1 definition without comment\n" . $herecurr); - } - } # check for memory barriers without a comment. - if ($line =~ /\b(mb|rmb|wmb|read_barrier_depends|smp_mb|smp_rmb|smp_wmb|smp_read_barrier_depends)\(/) { + if ($line =~ /\b(smp_mb|smp_rmb|smp_wmb|smp_read_barrier_depends)\(/) { if (!ctx_has_comment($first_line, $linenr)) { - CHK("memory barrier without comment\n" . $herecurr); + WARN("memory barrier without comment\n" . $herecurr); } } # check of hardware specific defines - if ($line =~ m@^.\s*\#\s*if.*\b(__i386__|__powerpc64__|__sun__|__s390x__)\b@ && $realfile !~ m@include/asm-@) { - CHK("architecture specific defines should be avoided\n" . $herecurr); +# we have e.g. CONFIG_LINUX and CONFIG_WIN32 for common cases +# where they might be necessary. + if ($line =~ m@^.\s*\#\s*if.*\b__@) { + WARN("architecture specific defines should be avoided\n" . $herecurr); } # Check that the storage class is at the beginning of a declaration @@ -2748,11 +2406,6 @@ sub process { ERROR("inline keyword should sit between storage class and type\n" . $herecurr); } -# Check for __inline__ and __inline, prefer inline - if ($line =~ /\b(__inline__|__inline)\b/) { - WARN("plain inline is preferred over $1\n" . $herecurr); - } - # check for sizeof(&) if ($line =~ /\bsizeof\s*\(\s*\&/) { WARN("sizeof(& should be avoided\n" . $herecurr); @@ -2785,98 +2438,55 @@ sub process { WARN("externs should be avoided in .c files\n" . $herecurr); } -# checks for new __setup's - if ($rawline =~ /\b__setup\("([^"]*)"/) { - my $name = $1; - - if (!grep(/$name/, @setup_docs)) { - CHK("__setup appears un-documented -- check Documentation/kernel-parameters.txt\n" . $herecurr); +# check for pointless casting of g_malloc return + if ($line =~ /\*\s*\)\s*g_(try)?(m|re)alloc(0?)(_n)?\b/) { + if ($2 == 'm') { + WARN("unnecessary cast may hide bugs, use g_$1new$3 instead\n" . $herecurr); + } else { + WARN("unnecessary cast may hide bugs, use g_$1renew$3 instead\n" . $herecurr); } } -# check for pointless casting of kmalloc return - if ($line =~ /\*\s*\)\s*k[czm]alloc\b/) { - WARN("unnecessary cast may hide bugs, see http://c-faq.com/malloc/mallocnocast.html\n" . $herecurr); - } - # check for gcc specific __FUNCTION__ if ($line =~ /__FUNCTION__/) { WARN("__func__ should be used instead of gcc specific __FUNCTION__\n" . $herecurr); } -# check for semaphores used as mutexes - if ($line =~ /^.\s*(DECLARE_MUTEX|init_MUTEX)\s*\(/) { - WARN("mutexes are preferred for single holder semaphores\n" . $herecurr); - } -# check for semaphores used as mutexes - if ($line =~ /^.\s*init_MUTEX_LOCKED\s*\(/) { - WARN("consider using a completion\n" . $herecurr); - - } -# recommend strict_strto* over simple_strto* - if ($line =~ /\bsimple_(strto.*?)\s*\(/) { - WARN("consider using strict_$1 in preference to simple_$1\n" . $herecurr); +# recommend qemu_strto* over strto* for numeric conversions + if ($line =~ /\b(strto[^k].*?)\s*\(/) { + WARN("consider using qemu_$1 in preference to $1\n" . $herecurr); } -# check for __initcall(), use device_initcall() explicitly please - if ($line =~ /^.\s*__initcall\s*\(/) { - WARN("please use device_initcall() instead of __initcall()\n" . $herecurr); +# check for module_init(), use category-specific init macros explicitly please + if ($line =~ /^module_init\s*\(/) { + WARN("please use block_init(), type_init() etc. instead of module_init()\n" . $herecurr); } # check for various ops structs, ensure they are const. - my $struct_ops = qr{acpi_dock_ops| - address_space_operations| - backlight_ops| - block_device_operations| - dentry_operations| - dev_pm_ops| - dma_map_ops| - extent_io_ops| - file_lock_operations| - file_operations| - hv_ops| - ide_dma_ops| - intel_dvo_dev_ops| - item_operations| - iwl_ops| - kgdb_arch| - kgdb_io| - kset_uevent_ops| - lock_manager_operations| - microcode_ops| - mtrr_ops| - neigh_ops| - nlmsvc_binding| - pci_raw_ops| - pipe_buf_operations| - platform_hibernation_ops| - platform_suspend_ops| - proto_ops| - rpc_pipe_ops| - seq_operations| - snd_ac97_build_ops| - soc_pcmcia_socket_ops| - stacktrace_ops| - sysfs_ops| - tty_operations| - usb_mon_operations| - wd_ops}x; + my $struct_ops = qr{AIOCBInfo| + BdrvActionOps| + BlockDevOps| + BlockJobDriver| + DisplayChangeListenerOps| + GraphicHwOps| + IDEDMAOps| + KVMCapabilityInfo| + MemoryRegionIOMMUOps| + MemoryRegionOps| + MemoryRegionPortio| + QEMUFileOps| + SCSIBusInfo| + SCSIReqOps| + Spice[A-Z][a-zA-Z0-9]*Interface| + TPMDriverOps| + USBDesc[A-Z][a-zA-Z0-9]*| + VhostOps| + VMStateDescription| + VMStateInfo}x; if ($line !~ /\bconst\b/ && - $line =~ /\bstruct\s+($struct_ops)\b/) { + $line =~ /\b($struct_ops)\b/) { WARN("struct $1 should normally be const\n" . $herecurr); } -# use of NR_CPUS is usually wrong -# ignore definitions of NR_CPUS and usage to define arrays as likely right - if ($line =~ /\bNR_CPUS\b/ && - $line !~ /^.\s*\s*#\s*if\b.*\bNR_CPUS\b/ && - $line !~ /^.\s*\s*#\s*define\b.*\bNR_CPUS\b/ && - $line !~ /^.\s*$Declare\s.*\[[^\]]*NR_CPUS[^\]]*\]/ && - $line !~ /\[[^\]]*\.\.\.[^\]]*NR_CPUS[^\]]*\]/ && - $line !~ /\[[^\]]*NR_CPUS[^\]]*\.\.\.[^\]]*\]/) - { - WARN("usage of NR_CPUS is often wrong - consider using cpu_possible(), num_possible_cpus(), for_each_possible_cpu(), etc\n" . $herecurr); - } - # check for %L{u,d,i} in strings my $string; while ($line =~ /(?:^|")([X\t]*)(?:"|$)/g) { @@ -2888,29 +2498,46 @@ sub process { } } -# whine mightly about in_atomic - if ($line =~ /\bin_atomic\s*\(/) { - if ($realfile =~ m@^drivers/@) { - ERROR("do not use in_atomic in drivers\n" . $herecurr); - } elsif ($realfile !~ m@^kernel/@) { - WARN("use of in_atomic() is incorrect outside core kernel code\n" . $herecurr); - } +# QEMU specific tests + if ($rawline =~ /\b(?:Qemu|QEmu)\b/) { + WARN("use QEMU instead of Qemu or QEmu\n" . $herecurr); } -# check for lockdep_set_novalidate_class - if ($line =~ /^.\s*lockdep_set_novalidate_class\s*\(/ || - $line =~ /__lockdep_no_validate__\s*\)/ ) { - if ($realfile !~ m@^kernel/lockdep@ && - $realfile !~ m@^include/linux/lockdep@ && - $realfile !~ m@^drivers/base/core@) { - ERROR("lockdep_no_validate class is reserved for device->mutex.\n" . $herecurr); - } +# Qemu error function tests + + # Find newlines in error messages + my $qemu_error_funcs = qr{error_setg| + error_setg_errno| + error_setg_win32| + error_set| + error_vreport| + error_report}x; + + if ($rawline =~ /\b(?:$qemu_error_funcs)\s*\(\s*\".*\\n/) { + WARN("Error messages should not contain newlines\n" . $herecurr); + } + + # Continue checking for error messages that contains newlines. This + # check handles cases where string literals are spread over multiple lines. + # Example: + # error_report("Error msg line #1" + # "Error msg line #2\n"); + my $quoted_newline_regex = qr{\+\s*\".*\\n.*\"}; + my $continued_str_literal = qr{\+\s*\".*\"}; + + if ($rawline =~ /$quoted_newline_regex/) { + # Backtrack to first line that does not contain only a quoted literal + # and assume that it is the start of the statement. + my $i = $linenr - 2; + + while (($i >= 0) & $rawlines[$i] =~ /$continued_str_literal/) { + $i--; } -# QEMU specific tests - if ($rawline =~ /\b(?:Qemu|QEmu)\b/) { - WARN("use QEMU instead of Qemu or QEmu\n" . $herecurr); + if ($rawlines[$i] =~ /\b(?:$qemu_error_funcs)\s*\(/) { + WARN("Error messages should not contain newlines\n" . $herecurr); } + } # check for non-portable ffs() calls that have portable alternatives in QEMU if ($line =~ /\bffs\(/) { diff --git a/qemu/scripts/clean-includes b/qemu/scripts/clean-includes new file mode 100755 index 000000000..72b47f17f --- /dev/null +++ b/qemu/scripts/clean-includes @@ -0,0 +1,165 @@ +#!/bin/sh -e +# +# Clean up QEMU #include lines by ensuring that qemu/osdep.h +# is the first include listed in .c files, and no headers provided +# by osdep.h itself are redundantly included in either .c or .h files. +# +# Copyright (c) 2015 Linaro Limited +# +# Authors: +# Peter Maydell <peter.maydell@linaro.org> +# +# This work is licensed under the terms of the GNU GPL, version 2 +# or (at your option) any later version. See the COPYING file in +# the top-level directory. + +# Usage: +# clean-includes [--git subjectprefix] file ... +# or +# clean-includes [--git subjectprefix] --all +# +# If the --git subjectprefix option is given, then after making +# the changes to the files this script will create a git commit +# with the subject line "subjectprefix: Clean up includes" +# and a boilerplate commit message. +# +# Using --all will cause clean-includes to run on the whole source +# tree (excluding certain directories which are known not to need +# handling). + +# This script requires Coccinelle to be installed. + +# .c files will have the osdep.h included added, and redundant +# includes removed. +# .h files will have redundant includes (including includes of osdep.h) +# removed. +# Other files (including C++ and ObjectiveC) can't be handled by this script. + +# The following one-liner may be handy for finding files to run this on. +# However some caution is required regarding files that might be part +# of the guest agent or standalone tests. + +# for i in `git ls-tree --name-only HEAD` ; do test -f $i && \ +# grep -E '^# *include' $i | head -1 | grep 'osdep.h' ; test $? != 0 && \ +# echo $i ; done + + +GIT=no + +# Extended regular expression defining files to ignore when using --all +XDIRREGEX='^(tests/tcg|tests/multiboot|pc-bios|disas/libvixl)' + +if [ $# -ne 0 ] && [ "$1" = "--git" ]; then + if [ $# -eq 1 ]; then + echo "--git option requires an argument" + exit 1 + fi + GITSUBJ="$2" + GIT=yes + shift + shift +fi + +if [ $# -eq 0 ]; then + echo "Usage: clean-includes [--git subjectprefix] [--all | foo.c ...]" + echo "(modifies the files in place)" + exit 1 +fi + +if [ "$1" = "--all" ]; then + # We assume there are no files in the tree with spaces in their name + set -- $(git ls-files '*.[ch]' | grep -E -v "$XDIRREGEX") +fi + +# Annoyingly coccinelle won't read a scriptfile unless its +# name ends '.cocci', so write it out to a tempfile with the +# right kind of name. +COCCIFILE="$(mktemp --suffix=.cocci)" + +trap 'rm -f -- "$COCCIFILE"' INT TERM HUP EXIT + +cat >"$COCCIFILE" <<EOT +@@ +@@ + +( ++ #include "qemu/osdep.h" + #include "..." +| ++ #include "qemu/osdep.h" + #include <...> +) +EOT + + +for f in "$@"; do + case "$f" in + *.inc.c) + # These aren't standalone C source files + echo "SKIPPING $f (not a standalone source file)" + continue + ;; + *.c) + MODE=c + ;; + *include/qemu/osdep.h | \ + *include/qemu/compiler.h | \ + *include/standard-headers/ ) + # Removing include lines from osdep.h itself would be counterproductive. + echo "SKIPPING $f (special case header)" + continue + ;; + *include/standard-headers/*) + echo "SKIPPING $f (autogenerated header)" + continue + ;; + *.h) + MODE=h + ;; + *) + echo "WARNING: ignoring $f (cannot handle non-C files)" + continue + ;; + esac + + if [ "$MODE" = "c" ]; then + # First, use Coccinelle to add qemu/osdep.h before the first existing include + # (this will add two lines if the file uses both "..." and <...> #includes, + # but we will remove the extras in the next step) + spatch --in-place --no-show-diff --cocci-file "$COCCIFILE" "$f" + + # Now remove any duplicate osdep.h includes + perl -n -i -e 'print if !/#include "qemu\/osdep.h"/ || !$n++;' "$f" + else + # Remove includes of osdep.h itself + perl -n -i -e 'print if !/\s*#\s*include\s*(["<][^>"]*[">])/ || + ! (grep { $_ eq $1 } qw ("qemu/osdep.h"))' "$f" + fi + + # Remove includes that osdep.h already provides + perl -n -i -e 'print if !/\s*#\s*include\s*(["<][^>"]*[">])/ || + ! (grep { $_ eq $1 } qw ( + "config-host.h" "config-target.h" "qemu/compiler.h" + <setjmp.h> <stdarg.h> <stddef.h> <stdbool.h> <stdint.h> <sys/types.h> + <stdlib.h> <stdio.h> <string.h> <strings.h> <inttypes.h> + <limits.h> <unistd.h> <time.h> <ctype.h> <errno.h> <fcntl.h> + <sys/stat.h> <sys/time.h> <assert.h> <signal.h> + "sysemu/os-posix.h, sysemu/os-win32.h "glib-compat.h" + "qemu/typedefs.h" + ))' "$f" + +done + +if [ "$GIT" = "yes" ]; then + git add -- "$@" + git commit --signoff -F - <<EOF +$GITSUBJ: Clean up includes + +Clean up includes so that osdep.h is included first and headers +which it implies are not included manually. + +This commit was created with scripts/clean-includes. + +EOF + +fi diff --git a/qemu/scripts/cocci-macro-file.h b/qemu/scripts/cocci-macro-file.h new file mode 100644 index 000000000..eceb4be73 --- /dev/null +++ b/qemu/scripts/cocci-macro-file.h @@ -0,0 +1,119 @@ +/* Macro file for Coccinelle + * + * Copyright (C) 2015 Red Hat, Inc. + * + * Authors: + * Paolo Bonzini <pbonzini@redhat.com> + * + * This work is licensed under the terms of the GNU GPL, version 2 or, at your + * option, any later version. See the COPYING file in the top-level directory. + */ + +/* Coccinelle only does limited parsing of headers, and chokes on some idioms + * defined in compiler.h and queue.h. Macros that Coccinelle must know about + * in order to parse .c files must be in a separate macro file---which is + * exactly what you're staring at now. + * + * To use this file, add the "--macro-file scripts/cocci-macro-file.h" to the + * Coccinelle command line. + */ + +/* From qemu/compiler.h */ +#define QEMU_GNUC_PREREQ(maj, min) 1 +#define QEMU_NORETURN __attribute__ ((__noreturn__)) +#define QEMU_WARN_UNUSED_RESULT __attribute__((warn_unused_result)) +#define QEMU_SENTINEL __attribute__((sentinel)) +#define QEMU_ARTIFICIAL __attribute__((always_inline, artificial)) +#define QEMU_PACKED __attribute__((gcc_struct, packed)) + +#define cat(x,y) x ## y +#define cat2(x,y) cat(x,y) +#define QEMU_BUILD_BUG_ON(x) \ + typedef char cat2(qemu_build_bug_on__,__LINE__)[(x)?-1:1] __attribute__((unused)); + +#define GCC_FMT_ATTR(n, m) __attribute__((format(gnu_printf, n, m))) + +#define xglue(x, y) x ## y +#define glue(x, y) xglue(x, y) +#define stringify(s) tostring(s) +#define tostring(s) #s + +#define typeof_field(type, field) typeof(((type *)0)->field) +#define type_check(t1,t2) ((t1*)0 - (t2*)0) + +/* From qemu/queue.h */ + +#define QLIST_HEAD(name, type) \ +struct name { \ + struct type *lh_first; /* first element */ \ +} + +#define QLIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define QLIST_ENTRY(type) \ +struct { \ + struct type *le_next; /* next element */ \ + struct type **le_prev; /* address of previous next element */ \ +} + +/* + * Singly-linked List definitions. + */ +#define QSLIST_HEAD(name, type) \ +struct name { \ + struct type *slh_first; /* first element */ \ +} + +#define QSLIST_HEAD_INITIALIZER(head) \ + { NULL } + +#define QSLIST_ENTRY(type) \ +struct { \ + struct type *sle_next; /* next element */ \ +} + +/* + * Simple queue definitions. + */ +#define QSIMPLEQ_HEAD(name, type) \ +struct name { \ + struct type *sqh_first; /* first element */ \ + struct type **sqh_last; /* addr of last next element */ \ +} + +#define QSIMPLEQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).sqh_first } + +#define QSIMPLEQ_ENTRY(type) \ +struct { \ + struct type *sqe_next; /* next element */ \ +} + +/* + * Tail queue definitions. + */ +#define Q_TAILQ_HEAD(name, type, qual) \ +struct name { \ + qual type *tqh_first; /* first element */ \ + qual type *qual *tqh_last; /* addr of last next element */ \ +} +#define QTAILQ_HEAD(name, type) \ +struct name { \ + type *tqh_first; /* first element */ \ + type **tqh_last; /* addr of last next element */ \ +} + +#define QTAILQ_HEAD_INITIALIZER(head) \ + { NULL, &(head).tqh_first } + +#define Q_TAILQ_ENTRY(type, qual) \ +struct { \ + qual type *tqe_next; /* next element */ \ + qual type *qual *tqe_prev; /* address of previous next element */\ +} +#define QTAILQ_ENTRY(type) \ +struct { \ + type *tqe_next; /* next element */ \ + type **tqe_prev; /* address of previous next element */ \ +} diff --git a/qemu/scripts/coverity-model.c b/qemu/scripts/coverity-model.c index 617f67d71..ee5bf9d07 100644 --- a/qemu/scripts/coverity-model.c +++ b/qemu/scripts/coverity-model.c @@ -236,6 +236,23 @@ void *g_try_realloc(void *ptr, size_t size) return g_try_realloc_n(ptr, 1, size); } +/* Other memory allocation functions */ + +void *g_memdup(const void *ptr, unsigned size) +{ + unsigned char *dup; + unsigned i; + + if (!ptr) { + return NULL; + } + + dup = g_malloc(size); + for (i = 0; i < size; i++) + dup[i] = ((unsigned char *)ptr)[i]; + return dup; +} + /* * GLib string allocation functions */ @@ -325,6 +342,15 @@ char *g_strconcat(const char *s, ...) /* Other glib functions */ +typedef struct pollfd GPollFD; + +int poll(); + +int g_poll (GPollFD *fds, unsigned nfds, int timeout) +{ + return poll(fds, nfds, timeout); +} + typedef struct _GIOChannel GIOChannel; GIOChannel *g_io_channel_unix_new(int fd) { diff --git a/qemu/scripts/create_config b/qemu/scripts/create_config index 546f88914..9cb176f1b 100755 --- a/qemu/scripts/create_config +++ b/qemu/scripts/create_config @@ -61,6 +61,15 @@ case $line in value=${line#*=} echo "#define $name $value" ;; + HAVE_*=y) # configuration + name=${line%=*} + echo "#define $name 1" + ;; + HAVE_*=*) # configuration + name=${line%=*} + value=${line#*=} + echo "#define $name $value" + ;; ARCH=*) # configuration arch=${line#*=} arch_name=`echo $arch | LC_ALL=C tr '[a-z]' '[A-Z]'` diff --git a/qemu/scripts/dump-guest-memory.py b/qemu/scripts/dump-guest-memory.py index dc8e44acf..c0a2e99f4 100644 --- a/qemu/scripts/dump-guest-memory.py +++ b/qemu/scripts/dump-guest-memory.py @@ -1,39 +1,456 @@ -# This python script adds a new gdb command, "dump-guest-memory". It -# should be loaded with "source dump-guest-memory.py" at the (gdb) -# prompt. -# -# Copyright (C) 2013, Red Hat, Inc. -# -# Authors: -# Laszlo Ersek <lersek@redhat.com> -# -# This work is licensed under the terms of the GNU GPL, version 2 or later. See -# the COPYING file in the top-level directory. -# +""" +This python script adds a new gdb command, "dump-guest-memory". It +should be loaded with "source dump-guest-memory.py" at the (gdb) +prompt. + +Copyright (C) 2013, Red Hat, Inc. + +Authors: + Laszlo Ersek <lersek@redhat.com> + Janosch Frank <frankja@linux.vnet.ibm.com> + +This work is licensed under the terms of the GNU GPL, version 2 or later. See +the COPYING file in the top-level directory. +""" + +import ctypes + +UINTPTR_T = gdb.lookup_type("uintptr_t") + +TARGET_PAGE_SIZE = 0x1000 +TARGET_PAGE_MASK = 0xFFFFFFFFFFFFF000 + +# Special value for e_phnum. This indicates that the real number of +# program headers is too large to fit into e_phnum. Instead the real +# value is in the field sh_info of section 0. +PN_XNUM = 0xFFFF + +EV_CURRENT = 1 + +ELFCLASS32 = 1 +ELFCLASS64 = 2 + +ELFDATA2LSB = 1 +ELFDATA2MSB = 2 + +ET_CORE = 4 + +PT_LOAD = 1 +PT_NOTE = 4 + +EM_386 = 3 +EM_PPC = 20 +EM_PPC64 = 21 +EM_S390 = 22 +EM_AARCH = 183 +EM_X86_64 = 62 + +class ELF(object): + """Representation of a ELF file.""" + + def __init__(self, arch): + self.ehdr = None + self.notes = [] + self.segments = [] + self.notes_size = 0 + self.endianess = None + self.elfclass = ELFCLASS64 + + if arch == 'aarch64-le': + self.endianess = ELFDATA2LSB + self.elfclass = ELFCLASS64 + self.ehdr = get_arch_ehdr(self.endianess, self.elfclass) + self.ehdr.e_machine = EM_AARCH + + elif arch == 'aarch64-be': + self.endianess = ELFDATA2MSB + self.ehdr = get_arch_ehdr(self.endianess, self.elfclass) + self.ehdr.e_machine = EM_AARCH + + elif arch == 'X86_64': + self.endianess = ELFDATA2LSB + self.ehdr = get_arch_ehdr(self.endianess, self.elfclass) + self.ehdr.e_machine = EM_X86_64 + + elif arch == '386': + self.endianess = ELFDATA2LSB + self.elfclass = ELFCLASS32 + self.ehdr = get_arch_ehdr(self.endianess, self.elfclass) + self.ehdr.e_machine = EM_386 + + elif arch == 's390': + self.endianess = ELFDATA2MSB + self.ehdr = get_arch_ehdr(self.endianess, self.elfclass) + self.ehdr.e_machine = EM_S390 + + elif arch == 'ppc64-le': + self.endianess = ELFDATA2LSB + self.ehdr = get_arch_ehdr(self.endianess, self.elfclass) + self.ehdr.e_machine = EM_PPC64 + + elif arch == 'ppc64-be': + self.endianess = ELFDATA2MSB + self.ehdr = get_arch_ehdr(self.endianess, self.elfclass) + self.ehdr.e_machine = EM_PPC64 + + else: + raise gdb.GdbError("No valid arch type specified.\n" + "Currently supported types:\n" + "aarch64-be, aarch64-le, X86_64, 386, s390, " + "ppc64-be, ppc64-le") + + self.add_segment(PT_NOTE, 0, 0) + + def add_note(self, n_name, n_desc, n_type): + """Adds a note to the ELF.""" + + note = get_arch_note(self.endianess, len(n_name), len(n_desc)) + note.n_namesz = len(n_name) + 1 + note.n_descsz = len(n_desc) + note.n_name = n_name.encode() + note.n_type = n_type + + # Desc needs to be 4 byte aligned (although the 64bit spec + # specifies 8 byte). When defining n_desc as uint32 it will be + # automatically aligned but we need the memmove to copy the + # string into it. + ctypes.memmove(note.n_desc, n_desc.encode(), len(n_desc)) + + self.notes.append(note) + self.segments[0].p_filesz += ctypes.sizeof(note) + self.segments[0].p_memsz += ctypes.sizeof(note) + + def add_segment(self, p_type, p_paddr, p_size): + """Adds a segment to the elf.""" + + phdr = get_arch_phdr(self.endianess, self.elfclass) + phdr.p_type = p_type + phdr.p_paddr = p_paddr + phdr.p_filesz = p_size + phdr.p_memsz = p_size + self.segments.append(phdr) + self.ehdr.e_phnum += 1 + + def to_file(self, elf_file): + """Writes all ELF structures to the the passed file. + + Structure: + Ehdr + Segment 0:PT_NOTE + Segment 1:PT_LOAD + Segment N:PT_LOAD + Note 0..N + Dump contents + """ + elf_file.write(self.ehdr) + off = ctypes.sizeof(self.ehdr) + \ + len(self.segments) * ctypes.sizeof(self.segments[0]) + + for phdr in self.segments: + phdr.p_offset = off + elf_file.write(phdr) + off += phdr.p_filesz + + for note in self.notes: + elf_file.write(note) + + +def get_arch_note(endianess, len_name, len_desc): + """Returns a Note class with the specified endianess.""" + + if endianess == ELFDATA2LSB: + superclass = ctypes.LittleEndianStructure + else: + superclass = ctypes.BigEndianStructure + + len_name = len_name + 1 + + class Note(superclass): + """Represents an ELF note, includes the content.""" + + _fields_ = [("n_namesz", ctypes.c_uint32), + ("n_descsz", ctypes.c_uint32), + ("n_type", ctypes.c_uint32), + ("n_name", ctypes.c_char * len_name), + ("n_desc", ctypes.c_uint32 * ((len_desc + 3) // 4))] + return Note() + + +class Ident(ctypes.Structure): + """Represents the ELF ident array in the ehdr structure.""" + + _fields_ = [('ei_mag0', ctypes.c_ubyte), + ('ei_mag1', ctypes.c_ubyte), + ('ei_mag2', ctypes.c_ubyte), + ('ei_mag3', ctypes.c_ubyte), + ('ei_class', ctypes.c_ubyte), + ('ei_data', ctypes.c_ubyte), + ('ei_version', ctypes.c_ubyte), + ('ei_osabi', ctypes.c_ubyte), + ('ei_abiversion', ctypes.c_ubyte), + ('ei_pad', ctypes.c_ubyte * 7)] + + def __init__(self, endianess, elfclass): + self.ei_mag0 = 0x7F + self.ei_mag1 = ord('E') + self.ei_mag2 = ord('L') + self.ei_mag3 = ord('F') + self.ei_class = elfclass + self.ei_data = endianess + self.ei_version = EV_CURRENT + + +def get_arch_ehdr(endianess, elfclass): + """Returns a EHDR64 class with the specified endianess.""" + + if endianess == ELFDATA2LSB: + superclass = ctypes.LittleEndianStructure + else: + superclass = ctypes.BigEndianStructure + + class EHDR64(superclass): + """Represents the 64 bit ELF header struct.""" + + _fields_ = [('e_ident', Ident), + ('e_type', ctypes.c_uint16), + ('e_machine', ctypes.c_uint16), + ('e_version', ctypes.c_uint32), + ('e_entry', ctypes.c_uint64), + ('e_phoff', ctypes.c_uint64), + ('e_shoff', ctypes.c_uint64), + ('e_flags', ctypes.c_uint32), + ('e_ehsize', ctypes.c_uint16), + ('e_phentsize', ctypes.c_uint16), + ('e_phnum', ctypes.c_uint16), + ('e_shentsize', ctypes.c_uint16), + ('e_shnum', ctypes.c_uint16), + ('e_shstrndx', ctypes.c_uint16)] + + def __init__(self): + super(superclass, self).__init__() + self.e_ident = Ident(endianess, elfclass) + self.e_type = ET_CORE + self.e_version = EV_CURRENT + self.e_ehsize = ctypes.sizeof(self) + self.e_phoff = ctypes.sizeof(self) + self.e_phentsize = ctypes.sizeof(get_arch_phdr(endianess, elfclass)) + self.e_phnum = 0 + + + class EHDR32(superclass): + """Represents the 32 bit ELF header struct.""" + + _fields_ = [('e_ident', Ident), + ('e_type', ctypes.c_uint16), + ('e_machine', ctypes.c_uint16), + ('e_version', ctypes.c_uint32), + ('e_entry', ctypes.c_uint32), + ('e_phoff', ctypes.c_uint32), + ('e_shoff', ctypes.c_uint32), + ('e_flags', ctypes.c_uint32), + ('e_ehsize', ctypes.c_uint16), + ('e_phentsize', ctypes.c_uint16), + ('e_phnum', ctypes.c_uint16), + ('e_shentsize', ctypes.c_uint16), + ('e_shnum', ctypes.c_uint16), + ('e_shstrndx', ctypes.c_uint16)] + + def __init__(self): + super(superclass, self).__init__() + self.e_ident = Ident(endianess, elfclass) + self.e_type = ET_CORE + self.e_version = EV_CURRENT + self.e_ehsize = ctypes.sizeof(self) + self.e_phoff = ctypes.sizeof(self) + self.e_phentsize = ctypes.sizeof(get_arch_phdr(endianess, elfclass)) + self.e_phnum = 0 + + # End get_arch_ehdr + if elfclass == ELFCLASS64: + return EHDR64() + else: + return EHDR32() + + +def get_arch_phdr(endianess, elfclass): + """Returns a 32 or 64 bit PHDR class with the specified endianess.""" + + if endianess == ELFDATA2LSB: + superclass = ctypes.LittleEndianStructure + else: + superclass = ctypes.BigEndianStructure + + class PHDR64(superclass): + """Represents the 64 bit ELF program header struct.""" + + _fields_ = [('p_type', ctypes.c_uint32), + ('p_flags', ctypes.c_uint32), + ('p_offset', ctypes.c_uint64), + ('p_vaddr', ctypes.c_uint64), + ('p_paddr', ctypes.c_uint64), + ('p_filesz', ctypes.c_uint64), + ('p_memsz', ctypes.c_uint64), + ('p_align', ctypes.c_uint64)] + + class PHDR32(superclass): + """Represents the 32 bit ELF program header struct.""" + + _fields_ = [('p_type', ctypes.c_uint32), + ('p_offset', ctypes.c_uint32), + ('p_vaddr', ctypes.c_uint32), + ('p_paddr', ctypes.c_uint32), + ('p_filesz', ctypes.c_uint32), + ('p_memsz', ctypes.c_uint32), + ('p_flags', ctypes.c_uint32), + ('p_align', ctypes.c_uint32)] + + # End get_arch_phdr + if elfclass == ELFCLASS64: + return PHDR64() + else: + return PHDR32() + + +def int128_get64(val): + """Returns low 64bit part of Int128 struct.""" + + assert val["hi"] == 0 + return val["lo"] + + +def qlist_foreach(head, field_str): + """Generator for qlists.""" + + var_p = head["lh_first"] + while var_p != 0: + var = var_p.dereference() + var_p = var[field_str]["le_next"] + yield var + + +def qemu_get_ram_block(ram_addr): + """Returns the RAMBlock struct to which the given address belongs.""" + + ram_blocks = gdb.parse_and_eval("ram_list.blocks") + + for block in qlist_foreach(ram_blocks, "next"): + if (ram_addr - block["offset"]) < block["used_length"]: + return block + + raise gdb.GdbError("Bad ram offset %x" % ram_addr) + + +def qemu_get_ram_ptr(ram_addr): + """Returns qemu vaddr for given guest physical address.""" + + block = qemu_get_ram_block(ram_addr) + return block["host"] + (ram_addr - block["offset"]) + + +def memory_region_get_ram_ptr(memory_region): + if memory_region["alias"] != 0: + return (memory_region_get_ram_ptr(memory_region["alias"].dereference()) + + memory_region["alias_offset"]) + + return qemu_get_ram_ptr(memory_region["ram_block"]["offset"]) + + +def get_guest_phys_blocks(): + """Returns a list of ram blocks. + + Each block entry contains: + 'target_start': guest block phys start address + 'target_end': guest block phys end address + 'host_addr': qemu vaddr of the block's start + """ + + guest_phys_blocks = [] + + print("guest RAM blocks:") + print("target_start target_end host_addr message " + "count") + print("---------------- ---------------- ---------------- ------- " + "-----") + + current_map_p = gdb.parse_and_eval("address_space_memory.current_map") + current_map = current_map_p.dereference() + + # Conversion to int is needed for python 3 + # compatibility. Otherwise range doesn't cast the value itself and + # breaks. + for cur in range(int(current_map["nr"])): + flat_range = (current_map["ranges"] + cur).dereference() + memory_region = flat_range["mr"].dereference() + + # we only care about RAM + if not memory_region["ram"]: + continue + + section_size = int128_get64(flat_range["addr"]["size"]) + target_start = int128_get64(flat_range["addr"]["start"]) + target_end = target_start + section_size + host_addr = (memory_region_get_ram_ptr(memory_region) + + flat_range["offset_in_region"]) + predecessor = None + + # find continuity in guest physical address space + if len(guest_phys_blocks) > 0: + predecessor = guest_phys_blocks[-1] + predecessor_size = (predecessor["target_end"] - + predecessor["target_start"]) + + # the memory API guarantees monotonically increasing + # traversal + assert predecessor["target_end"] <= target_start + + # we want continuity in both guest-physical and + # host-virtual memory + if (predecessor["target_end"] < target_start or + predecessor["host_addr"] + predecessor_size != host_addr): + predecessor = None + + if predecessor is None: + # isolated mapping, add it to the list + guest_phys_blocks.append({"target_start": target_start, + "target_end": target_end, + "host_addr": host_addr}) + message = "added" + else: + # expand predecessor until @target_end; predecessor's + # start doesn't change + predecessor["target_end"] = target_end + message = "joined" + + print("%016x %016x %016x %-7s %5u" % + (target_start, target_end, host_addr.cast(UINTPTR_T), + message, len(guest_phys_blocks))) + + return guest_phys_blocks + + # The leading docstring doesn't have idiomatic Python formatting. It is # printed by gdb's "help" command (the first line is printed in the # "help data" summary), and it should match how other help texts look in # gdb. - -import struct - class DumpGuestMemory(gdb.Command): """Extract guest vmcore from qemu process coredump. -The sole argument is FILE, identifying the target file to write the -guest vmcore to. +The two required arguments are FILE and ARCH: +FILE identifies the target file to write the guest vmcore to. +ARCH specifies the architecture for which the core will be generated. This GDB command reimplements the dump-guest-memory QMP command in python, using the representation of guest memory as captured in the qemu coredump. The qemu process that has been dumped must have had the -command line option "-machine dump-guest-core=on". +command line option "-machine dump-guest-core=on" which is the default. For simplicity, the "paging", "begin" and "end" parameters of the QMP command are not supported -- no attempt is made to get the guest's internal paging structures (ie. paging=false is hard-wired), and guest memory is always fully dumped. -Only x86_64 guests are supported. +Currently aarch64-be, aarch64-le, X86_64, 386, s390, ppc64-be, +ppc64-le guests are supported. The CORE/NT_PRSTATUS and QEMU notes (that is, the VCPUs' statuses) are not written to the vmcore. Preparing these would require context that is @@ -47,293 +464,66 @@ deliberately called abort(), or it was dumped in response to a signal at a halfway fortunate point, then its coredump should be in reasonable shape and this command should mostly work.""" - TARGET_PAGE_SIZE = 0x1000 - TARGET_PAGE_MASK = 0xFFFFFFFFFFFFF000 - - # Various ELF constants - EM_X86_64 = 62 # AMD x86-64 target machine - ELFDATA2LSB = 1 # little endian - ELFCLASS64 = 2 - ELFMAG = "\x7FELF" - EV_CURRENT = 1 - ET_CORE = 4 - PT_LOAD = 1 - PT_NOTE = 4 - - # Special value for e_phnum. This indicates that the real number of - # program headers is too large to fit into e_phnum. Instead the real - # value is in the field sh_info of section 0. - PN_XNUM = 0xFFFF - - # Format strings for packing and header size calculation. - ELF64_EHDR = ("4s" # e_ident/magic - "B" # e_ident/class - "B" # e_ident/data - "B" # e_ident/version - "B" # e_ident/osabi - "8s" # e_ident/pad - "H" # e_type - "H" # e_machine - "I" # e_version - "Q" # e_entry - "Q" # e_phoff - "Q" # e_shoff - "I" # e_flags - "H" # e_ehsize - "H" # e_phentsize - "H" # e_phnum - "H" # e_shentsize - "H" # e_shnum - "H" # e_shstrndx - ) - ELF64_PHDR = ("I" # p_type - "I" # p_flags - "Q" # p_offset - "Q" # p_vaddr - "Q" # p_paddr - "Q" # p_filesz - "Q" # p_memsz - "Q" # p_align - ) - def __init__(self): super(DumpGuestMemory, self).__init__("dump-guest-memory", gdb.COMMAND_DATA, gdb.COMPLETE_FILENAME) - self.uintptr_t = gdb.lookup_type("uintptr_t") - self.elf64_ehdr_le = struct.Struct("<%s" % self.ELF64_EHDR) - self.elf64_phdr_le = struct.Struct("<%s" % self.ELF64_PHDR) - - def int128_get64(self, val): - assert (val["hi"] == 0) - return val["lo"] - - def qlist_foreach(self, head, field_str): - var_p = head["lh_first"] - while (var_p != 0): - var = var_p.dereference() - yield var - var_p = var[field_str]["le_next"] - - def qemu_get_ram_block(self, ram_addr): - ram_blocks = gdb.parse_and_eval("ram_list.blocks") - for block in self.qlist_foreach(ram_blocks, "next"): - if (ram_addr - block["offset"] < block["length"]): - return block - raise gdb.GdbError("Bad ram offset %x" % ram_addr) - - def qemu_get_ram_ptr(self, ram_addr): - block = self.qemu_get_ram_block(ram_addr) - return block["host"] + (ram_addr - block["offset"]) - - def memory_region_get_ram_ptr(self, mr): - if (mr["alias"] != 0): - return (self.memory_region_get_ram_ptr(mr["alias"].dereference()) + - mr["alias_offset"]) - return self.qemu_get_ram_ptr(mr["ram_addr"] & self.TARGET_PAGE_MASK) - - def guest_phys_blocks_init(self): - self.guest_phys_blocks = [] - - def guest_phys_blocks_append(self): - print "guest RAM blocks:" - print ("target_start target_end host_addr message " - "count") - print ("---------------- ---------------- ---------------- ------- " - "-----") - - current_map_p = gdb.parse_and_eval("address_space_memory.current_map") - current_map = current_map_p.dereference() - for cur in range(current_map["nr"]): - flat_range = (current_map["ranges"] + cur).dereference() - mr = flat_range["mr"].dereference() - - # we only care about RAM - if (not mr["ram"]): - continue - - section_size = self.int128_get64(flat_range["addr"]["size"]) - target_start = self.int128_get64(flat_range["addr"]["start"]) - target_end = target_start + section_size - host_addr = (self.memory_region_get_ram_ptr(mr) + - flat_range["offset_in_region"]) - predecessor = None - - # find continuity in guest physical address space - if (len(self.guest_phys_blocks) > 0): - predecessor = self.guest_phys_blocks[-1] - predecessor_size = (predecessor["target_end"] - - predecessor["target_start"]) - - # the memory API guarantees monotonically increasing - # traversal - assert (predecessor["target_end"] <= target_start) - - # we want continuity in both guest-physical and - # host-virtual memory - if (predecessor["target_end"] < target_start or - predecessor["host_addr"] + predecessor_size != host_addr): - predecessor = None - - if (predecessor is None): - # isolated mapping, add it to the list - self.guest_phys_blocks.append({"target_start": target_start, - "target_end" : target_end, - "host_addr" : host_addr}) - message = "added" - else: - # expand predecessor until @target_end; predecessor's - # start doesn't change - predecessor["target_end"] = target_end - message = "joined" - - print ("%016x %016x %016x %-7s %5u" % - (target_start, target_end, host_addr.cast(self.uintptr_t), - message, len(self.guest_phys_blocks))) - - def cpu_get_dump_info(self): - # We can't synchronize the registers with KVM post-mortem, and - # the bits in (first_x86_cpu->env.hflags) seem to be stale; they - # may not reflect long mode for example. Hence just assume the - # most common values. This also means that instruction pointer - # etc. will be bogus in the dump, but at least the RAM contents - # should be valid. - self.dump_info = {"d_machine": self.EM_X86_64, - "d_endian" : self.ELFDATA2LSB, - "d_class" : self.ELFCLASS64} - - def encode_elf64_ehdr_le(self): - return self.elf64_ehdr_le.pack( - self.ELFMAG, # e_ident/magic - self.dump_info["d_class"], # e_ident/class - self.dump_info["d_endian"], # e_ident/data - self.EV_CURRENT, # e_ident/version - 0, # e_ident/osabi - "", # e_ident/pad - self.ET_CORE, # e_type - self.dump_info["d_machine"], # e_machine - self.EV_CURRENT, # e_version - 0, # e_entry - self.elf64_ehdr_le.size, # e_phoff - 0, # e_shoff - 0, # e_flags - self.elf64_ehdr_le.size, # e_ehsize - self.elf64_phdr_le.size, # e_phentsize - self.phdr_num, # e_phnum - 0, # e_shentsize - 0, # e_shnum - 0 # e_shstrndx - ) - - def encode_elf64_note_le(self): - return self.elf64_phdr_le.pack(self.PT_NOTE, # p_type - 0, # p_flags - (self.memory_offset - - len(self.note)), # p_offset - 0, # p_vaddr - 0, # p_paddr - len(self.note), # p_filesz - len(self.note), # p_memsz - 0 # p_align - ) - - def encode_elf64_load_le(self, offset, start_hwaddr, range_size): - return self.elf64_phdr_le.pack(self.PT_LOAD, # p_type - 0, # p_flags - offset, # p_offset - 0, # p_vaddr - start_hwaddr, # p_paddr - range_size, # p_filesz - range_size, # p_memsz - 0 # p_align - ) - - def note_init(self, name, desc, type): - # name must include a trailing NUL - namesz = (len(name) + 1 + 3) / 4 * 4 - descsz = (len(desc) + 3) / 4 * 4 - fmt = ("<" # little endian - "I" # n_namesz - "I" # n_descsz - "I" # n_type - "%us" # name - "%us" # desc - % (namesz, descsz)) - self.note = struct.pack(fmt, - len(name) + 1, len(desc), type, name, desc) - - def dump_init(self): - self.guest_phys_blocks_init() - self.guest_phys_blocks_append() - self.cpu_get_dump_info() - # we have no way to retrieve the VCPU status from KVM - # post-mortem - self.note_init("NONE", "EMPTY", 0) - - # Account for PT_NOTE. - self.phdr_num = 1 - - # We should never reach PN_XNUM for paging=false dumps: there's - # just a handful of discontiguous ranges after merging. - self.phdr_num += len(self.guest_phys_blocks) - assert (self.phdr_num < self.PN_XNUM) - - # Calculate the ELF file offset where the memory dump commences: - # - # ELF header - # PT_NOTE - # PT_LOAD: 1 - # PT_LOAD: 2 - # ... - # PT_LOAD: len(self.guest_phys_blocks) - # ELF note - # memory dump - self.memory_offset = (self.elf64_ehdr_le.size + - self.elf64_phdr_le.size * self.phdr_num + - len(self.note)) - - def dump_begin(self, vmcore): - vmcore.write(self.encode_elf64_ehdr_le()) - vmcore.write(self.encode_elf64_note_le()) - running = self.memory_offset + self.elf = None + self.guest_phys_blocks = None + + def dump_init(self, vmcore): + """Prepares and writes ELF structures to core file.""" + + # Needed to make crash happy, data for more useful notes is + # not available in a qemu core. + self.elf.add_note("NONE", "EMPTY", 0) + + # We should never reach PN_XNUM for paging=false dumps, + # there's just a handful of discontiguous ranges after + # merging. + # The constant is needed to account for the PT_NOTE segment. + phdr_num = len(self.guest_phys_blocks) + 1 + assert phdr_num < PN_XNUM + for block in self.guest_phys_blocks: - range_size = block["target_end"] - block["target_start"] - vmcore.write(self.encode_elf64_load_le(running, - block["target_start"], - range_size)) - running += range_size - vmcore.write(self.note) + block_size = block["target_end"] - block["target_start"] + self.elf.add_segment(PT_LOAD, block["target_start"], block_size) + + self.elf.to_file(vmcore) def dump_iterate(self, vmcore): + """Writes guest core to file.""" + qemu_core = gdb.inferiors()[0] for block in self.guest_phys_blocks: - cur = block["host_addr"] + cur = block["host_addr"] left = block["target_end"] - block["target_start"] - print ("dumping range at %016x for length %016x" % - (cur.cast(self.uintptr_t), left)) - while (left > 0): - chunk_size = min(self.TARGET_PAGE_SIZE, left) + print("dumping range at %016x for length %016x" % + (cur.cast(UINTPTR_T), left)) + + while left > 0: + chunk_size = min(TARGET_PAGE_SIZE, left) chunk = qemu_core.read_memory(cur, chunk_size) vmcore.write(chunk) - cur += chunk_size + cur += chunk_size left -= chunk_size - def create_vmcore(self, filename): - vmcore = open(filename, "wb") - self.dump_begin(vmcore) - self.dump_iterate(vmcore) - vmcore.close() - def invoke(self, args, from_tty): + """Handles command invocation from gdb.""" + # Unwittingly pressing the Enter key after the command should # not dump the same multi-gig coredump to the same file. self.dont_repeat() argv = gdb.string_to_argv(args) - if (len(argv) != 1): - raise gdb.GdbError("usage: dump-guest-memory FILE") + if len(argv) != 2: + raise gdb.GdbError("usage: dump-guest-memory FILE ARCH") + + self.elf = ELF(argv[1]) + self.guest_phys_blocks = get_guest_phys_blocks() - self.dump_init() - self.create_vmcore(argv[0]) + with open(argv[0], "wb") as vmcore: + self.dump_init(vmcore) + self.dump_iterate(vmcore) DumpGuestMemory() diff --git a/qemu/scripts/feature_to_c.sh b/qemu/scripts/feature_to_c.sh index 888548e58..fb1f3363f 100644 --- a/qemu/scripts/feature_to_c.sh +++ b/qemu/scripts/feature_to_c.sh @@ -36,7 +36,7 @@ for input; do arrayname=xml_feature_`echo $input | sed 's,.*/,,; s/[-.]/_/g'` ${AWK:-awk} 'BEGIN { n = 0 - printf "#include \"config.h\"\n" + printf "#include \"qemu/osdep.h\"\n" printf "#include \"qemu-common.h\"\n" printf "#include \"exec/gdbstub.h\"\n" print "static const char '$arrayname'[] = {" diff --git a/qemu/scripts/get_maintainer.pl b/qemu/scripts/get_maintainer.pl index f39630eb3..8261bcb1a 100755 --- a/qemu/scripts/get_maintainer.pl +++ b/qemu/scripts/get_maintainer.pl @@ -258,7 +258,7 @@ open (my $maint, '<', "${lk_path}MAINTAINERS") while (<$maint>) { my $line = $_; - if ($line =~ m/^(\C):\s*(.*)/) { + if ($line =~ m/^(.):\s*(.*)/) { my $type = $1; my $value = $2; @@ -493,7 +493,7 @@ sub range_is_maintained { for (my $i = $start; $i < $end; $i++) { my $line = $typevalue[$i]; - if ($line =~ m/^(\C):\s*(.*)/) { + if ($line =~ m/^(.):\s*(.*)/) { my $type = $1; my $value = $2; if ($type eq 'S') { @@ -511,7 +511,7 @@ sub range_has_maintainer { for (my $i = $start; $i < $end; $i++) { my $line = $typevalue[$i]; - if ($line =~ m/^(\C):\s*(.*)/) { + if ($line =~ m/^(.):\s*(.*)/) { my $type = $1; my $value = $2; if ($type eq 'M') { @@ -560,7 +560,7 @@ sub get_maintainers { for ($i = $start; $i < $end; $i++) { my $line = $typevalue[$i]; - if ($line =~ m/^(\C):\s*(.*)/) { + if ($line =~ m/^(.):\s*(.*)/) { my $type = $1; my $value = $2; if ($type eq 'X') { @@ -575,7 +575,7 @@ sub get_maintainers { if (!$exclude) { for ($i = $start; $i < $end; $i++) { my $line = $typevalue[$i]; - if ($line =~ m/^(\C):\s*(.*)/) { + if ($line =~ m/^(.):\s*(.*)/) { my $type = $1; my $value = $2; if ($type eq 'F') { @@ -636,7 +636,7 @@ sub get_maintainers { if ($email) { if (! $interactive) { - $email_git_fallback = 0 if @email_to > 0 || @list_to > 0 || $email_git || $email_git_blame; + $email_git_fallback = 0 if @email_to > 0 || $email_git || $email_git_blame; if ($email_git_fallback) { print STDERR "get_maintainer.pl: No maintainers found, printing recent contributors.\n"; print STDERR "get_maintainer.pl: Do not blindly cc: them on patches! Use common sense.\n"; @@ -855,7 +855,7 @@ sub find_first_section { while ($index < @typevalue) { my $tv = $typevalue[$index]; - if (($tv =~ m/^(\C):\s*(.*)/)) { + if (($tv =~ m/^(.):\s*(.*)/)) { last; } $index++; @@ -869,7 +869,7 @@ sub find_starting_index { while ($index > 0) { my $tv = $typevalue[$index]; - if (!($tv =~ m/^(\C):\s*(.*)/)) { + if (!($tv =~ m/^(.):\s*(.*)/)) { last; } $index--; @@ -883,7 +883,7 @@ sub find_ending_index { while ($index < @typevalue) { my $tv = $typevalue[$index]; - if (!($tv =~ m/^(\C):\s*(.*)/)) { + if (!($tv =~ m/^(.):\s*(.*)/)) { last; } $index++; @@ -909,7 +909,7 @@ sub get_maintainer_role { for ($i = $start + 1; $i < $end; $i++) { my $tv = $typevalue[$i]; - if ($tv =~ m/^(\C):\s*(.*)/) { + if ($tv =~ m/^(.):\s*(.*)/) { my $ptype = $1; my $pvalue = $2; if ($ptype eq "S") { @@ -968,7 +968,7 @@ sub add_categories { for ($i = $start + 1; $i < $end; $i++) { my $tv = $typevalue[$i]; - if ($tv =~ m/^(\C):\s*(.*)/) { + if ($tv =~ m/^(.):\s*(.*)/) { my $ptype = $1; my $pvalue = $2; if ($ptype eq "L") { @@ -1010,7 +1010,7 @@ sub add_categories { if ($name eq "") { if ($i > 0) { my $tv = $typevalue[$i - 1]; - if ($tv =~ m/^(\C):\s*(.*)/) { + if ($tv =~ m/^(.):\s*(.*)/) { if ($1 eq "P") { $name = $2; $pvalue = format_email($name, $address, $email_usename); diff --git a/qemu/scripts/kvm/kvm_stat b/qemu/scripts/kvm/kvm_stat index 7e5d25612..769d884b6 100755 --- a/qemu/scripts/kvm/kvm_stat +++ b/qemu/scripts/kvm/kvm_stat @@ -12,285 +12,312 @@ # the COPYING file in the top-level directory. import curses -import sys, os, time, optparse, ctypes -from ctypes import * - -class DebugfsProvider(object): - def __init__(self): - self.base = '/sys/kernel/debug/kvm' - self._fields = os.listdir(self.base) - def fields(self): - return self._fields - def select(self, fields): - self._fields = fields - def read(self): - def val(key): - return int(file(self.base + '/' + key).read()) - return dict([(key, val(key)) for key in self._fields]) - -vmx_exit_reasons = { - 0: 'EXCEPTION_NMI', - 1: 'EXTERNAL_INTERRUPT', - 2: 'TRIPLE_FAULT', - 7: 'PENDING_INTERRUPT', - 8: 'NMI_WINDOW', - 9: 'TASK_SWITCH', - 10: 'CPUID', - 12: 'HLT', - 14: 'INVLPG', - 15: 'RDPMC', - 16: 'RDTSC', - 18: 'VMCALL', - 19: 'VMCLEAR', - 20: 'VMLAUNCH', - 21: 'VMPTRLD', - 22: 'VMPTRST', - 23: 'VMREAD', - 24: 'VMRESUME', - 25: 'VMWRITE', - 26: 'VMOFF', - 27: 'VMON', - 28: 'CR_ACCESS', - 29: 'DR_ACCESS', - 30: 'IO_INSTRUCTION', - 31: 'MSR_READ', - 32: 'MSR_WRITE', - 33: 'INVALID_STATE', - 36: 'MWAIT_INSTRUCTION', - 39: 'MONITOR_INSTRUCTION', - 40: 'PAUSE_INSTRUCTION', - 41: 'MCE_DURING_VMENTRY', - 43: 'TPR_BELOW_THRESHOLD', - 44: 'APIC_ACCESS', - 48: 'EPT_VIOLATION', - 49: 'EPT_MISCONFIG', - 54: 'WBINVD', - 55: 'XSETBV', - 56: 'APIC_WRITE', - 58: 'INVPCID', +import sys +import os +import time +import optparse +import ctypes +import fcntl +import resource +import struct +import re +from collections import defaultdict +from time import sleep + +VMX_EXIT_REASONS = { + 'EXCEPTION_NMI': 0, + 'EXTERNAL_INTERRUPT': 1, + 'TRIPLE_FAULT': 2, + 'PENDING_INTERRUPT': 7, + 'NMI_WINDOW': 8, + 'TASK_SWITCH': 9, + 'CPUID': 10, + 'HLT': 12, + 'INVLPG': 14, + 'RDPMC': 15, + 'RDTSC': 16, + 'VMCALL': 18, + 'VMCLEAR': 19, + 'VMLAUNCH': 20, + 'VMPTRLD': 21, + 'VMPTRST': 22, + 'VMREAD': 23, + 'VMRESUME': 24, + 'VMWRITE': 25, + 'VMOFF': 26, + 'VMON': 27, + 'CR_ACCESS': 28, + 'DR_ACCESS': 29, + 'IO_INSTRUCTION': 30, + 'MSR_READ': 31, + 'MSR_WRITE': 32, + 'INVALID_STATE': 33, + 'MWAIT_INSTRUCTION': 36, + 'MONITOR_INSTRUCTION': 39, + 'PAUSE_INSTRUCTION': 40, + 'MCE_DURING_VMENTRY': 41, + 'TPR_BELOW_THRESHOLD': 43, + 'APIC_ACCESS': 44, + 'EPT_VIOLATION': 48, + 'EPT_MISCONFIG': 49, + 'WBINVD': 54, + 'XSETBV': 55, + 'APIC_WRITE': 56, + 'INVPCID': 58, } -svm_exit_reasons = { - 0x000: 'READ_CR0', - 0x003: 'READ_CR3', - 0x004: 'READ_CR4', - 0x008: 'READ_CR8', - 0x010: 'WRITE_CR0', - 0x013: 'WRITE_CR3', - 0x014: 'WRITE_CR4', - 0x018: 'WRITE_CR8', - 0x020: 'READ_DR0', - 0x021: 'READ_DR1', - 0x022: 'READ_DR2', - 0x023: 'READ_DR3', - 0x024: 'READ_DR4', - 0x025: 'READ_DR5', - 0x026: 'READ_DR6', - 0x027: 'READ_DR7', - 0x030: 'WRITE_DR0', - 0x031: 'WRITE_DR1', - 0x032: 'WRITE_DR2', - 0x033: 'WRITE_DR3', - 0x034: 'WRITE_DR4', - 0x035: 'WRITE_DR5', - 0x036: 'WRITE_DR6', - 0x037: 'WRITE_DR7', - 0x040: 'EXCP_BASE', - 0x060: 'INTR', - 0x061: 'NMI', - 0x062: 'SMI', - 0x063: 'INIT', - 0x064: 'VINTR', - 0x065: 'CR0_SEL_WRITE', - 0x066: 'IDTR_READ', - 0x067: 'GDTR_READ', - 0x068: 'LDTR_READ', - 0x069: 'TR_READ', - 0x06a: 'IDTR_WRITE', - 0x06b: 'GDTR_WRITE', - 0x06c: 'LDTR_WRITE', - 0x06d: 'TR_WRITE', - 0x06e: 'RDTSC', - 0x06f: 'RDPMC', - 0x070: 'PUSHF', - 0x071: 'POPF', - 0x072: 'CPUID', - 0x073: 'RSM', - 0x074: 'IRET', - 0x075: 'SWINT', - 0x076: 'INVD', - 0x077: 'PAUSE', - 0x078: 'HLT', - 0x079: 'INVLPG', - 0x07a: 'INVLPGA', - 0x07b: 'IOIO', - 0x07c: 'MSR', - 0x07d: 'TASK_SWITCH', - 0x07e: 'FERR_FREEZE', - 0x07f: 'SHUTDOWN', - 0x080: 'VMRUN', - 0x081: 'VMMCALL', - 0x082: 'VMLOAD', - 0x083: 'VMSAVE', - 0x084: 'STGI', - 0x085: 'CLGI', - 0x086: 'SKINIT', - 0x087: 'RDTSCP', - 0x088: 'ICEBP', - 0x089: 'WBINVD', - 0x08a: 'MONITOR', - 0x08b: 'MWAIT', - 0x08c: 'MWAIT_COND', - 0x08d: 'XSETBV', - 0x400: 'NPF', +SVM_EXIT_REASONS = { + 'READ_CR0': 0x000, + 'READ_CR3': 0x003, + 'READ_CR4': 0x004, + 'READ_CR8': 0x008, + 'WRITE_CR0': 0x010, + 'WRITE_CR3': 0x013, + 'WRITE_CR4': 0x014, + 'WRITE_CR8': 0x018, + 'READ_DR0': 0x020, + 'READ_DR1': 0x021, + 'READ_DR2': 0x022, + 'READ_DR3': 0x023, + 'READ_DR4': 0x024, + 'READ_DR5': 0x025, + 'READ_DR6': 0x026, + 'READ_DR7': 0x027, + 'WRITE_DR0': 0x030, + 'WRITE_DR1': 0x031, + 'WRITE_DR2': 0x032, + 'WRITE_DR3': 0x033, + 'WRITE_DR4': 0x034, + 'WRITE_DR5': 0x035, + 'WRITE_DR6': 0x036, + 'WRITE_DR7': 0x037, + 'EXCP_BASE': 0x040, + 'INTR': 0x060, + 'NMI': 0x061, + 'SMI': 0x062, + 'INIT': 0x063, + 'VINTR': 0x064, + 'CR0_SEL_WRITE': 0x065, + 'IDTR_READ': 0x066, + 'GDTR_READ': 0x067, + 'LDTR_READ': 0x068, + 'TR_READ': 0x069, + 'IDTR_WRITE': 0x06a, + 'GDTR_WRITE': 0x06b, + 'LDTR_WRITE': 0x06c, + 'TR_WRITE': 0x06d, + 'RDTSC': 0x06e, + 'RDPMC': 0x06f, + 'PUSHF': 0x070, + 'POPF': 0x071, + 'CPUID': 0x072, + 'RSM': 0x073, + 'IRET': 0x074, + 'SWINT': 0x075, + 'INVD': 0x076, + 'PAUSE': 0x077, + 'HLT': 0x078, + 'INVLPG': 0x079, + 'INVLPGA': 0x07a, + 'IOIO': 0x07b, + 'MSR': 0x07c, + 'TASK_SWITCH': 0x07d, + 'FERR_FREEZE': 0x07e, + 'SHUTDOWN': 0x07f, + 'VMRUN': 0x080, + 'VMMCALL': 0x081, + 'VMLOAD': 0x082, + 'VMSAVE': 0x083, + 'STGI': 0x084, + 'CLGI': 0x085, + 'SKINIT': 0x086, + 'RDTSCP': 0x087, + 'ICEBP': 0x088, + 'WBINVD': 0x089, + 'MONITOR': 0x08a, + 'MWAIT': 0x08b, + 'MWAIT_COND': 0x08c, + 'XSETBV': 0x08d, + 'NPF': 0x400, } # EC definition of HSR (from arch/arm64/include/asm/kvm_arm.h) -aarch64_exit_reasons = { - 0x00: 'UNKNOWN', - 0x01: 'WFI', - 0x03: 'CP15_32', - 0x04: 'CP15_64', - 0x05: 'CP14_MR', - 0x06: 'CP14_LS', - 0x07: 'FP_ASIMD', - 0x08: 'CP10_ID', - 0x0C: 'CP14_64', - 0x0E: 'ILL_ISS', - 0x11: 'SVC32', - 0x12: 'HVC32', - 0x13: 'SMC32', - 0x15: 'SVC64', - 0x16: 'HVC64', - 0x17: 'SMC64', - 0x18: 'SYS64', - 0x20: 'IABT', - 0x21: 'IABT_HYP', - 0x22: 'PC_ALIGN', - 0x24: 'DABT', - 0x25: 'DABT_HYP', - 0x26: 'SP_ALIGN', - 0x28: 'FP_EXC32', - 0x2C: 'FP_EXC64', - 0x2F: 'SERROR', - 0x30: 'BREAKPT', - 0x31: 'BREAKPT_HYP', - 0x32: 'SOFTSTP', - 0x33: 'SOFTSTP_HYP', - 0x34: 'WATCHPT', - 0x35: 'WATCHPT_HYP', - 0x38: 'BKPT32', - 0x3A: 'VECTOR32', - 0x3C: 'BRK64', +AARCH64_EXIT_REASONS = { + 'UNKNOWN': 0x00, + 'WFI': 0x01, + 'CP15_32': 0x03, + 'CP15_64': 0x04, + 'CP14_MR': 0x05, + 'CP14_LS': 0x06, + 'FP_ASIMD': 0x07, + 'CP10_ID': 0x08, + 'CP14_64': 0x0C, + 'ILL_ISS': 0x0E, + 'SVC32': 0x11, + 'HVC32': 0x12, + 'SMC32': 0x13, + 'SVC64': 0x15, + 'HVC64': 0x16, + 'SMC64': 0x17, + 'SYS64': 0x18, + 'IABT': 0x20, + 'IABT_HYP': 0x21, + 'PC_ALIGN': 0x22, + 'DABT': 0x24, + 'DABT_HYP': 0x25, + 'SP_ALIGN': 0x26, + 'FP_EXC32': 0x28, + 'FP_EXC64': 0x2C, + 'SERROR': 0x2F, + 'BREAKPT': 0x30, + 'BREAKPT_HYP': 0x31, + 'SOFTSTP': 0x32, + 'SOFTSTP_HYP': 0x33, + 'WATCHPT': 0x34, + 'WATCHPT_HYP': 0x35, + 'BKPT32': 0x38, + 'VECTOR32': 0x3A, + 'BRK64': 0x3C, } # From include/uapi/linux/kvm.h, KVM_EXIT_xxx -userspace_exit_reasons = { - 0: 'UNKNOWN', - 1: 'EXCEPTION', - 2: 'IO', - 3: 'HYPERCALL', - 4: 'DEBUG', - 5: 'HLT', - 6: 'MMIO', - 7: 'IRQ_WINDOW_OPEN', - 8: 'SHUTDOWN', - 9: 'FAIL_ENTRY', - 10: 'INTR', - 11: 'SET_TPR', - 12: 'TPR_ACCESS', - 13: 'S390_SIEIC', - 14: 'S390_RESET', - 15: 'DCR', - 16: 'NMI', - 17: 'INTERNAL_ERROR', - 18: 'OSI', - 19: 'PAPR_HCALL', - 20: 'S390_UCONTROL', - 21: 'WATCHDOG', - 22: 'S390_TSCH', - 23: 'EPR', - 24: 'SYSTEM_EVENT', +USERSPACE_EXIT_REASONS = { + 'UNKNOWN': 0, + 'EXCEPTION': 1, + 'IO': 2, + 'HYPERCALL': 3, + 'DEBUG': 4, + 'HLT': 5, + 'MMIO': 6, + 'IRQ_WINDOW_OPEN': 7, + 'SHUTDOWN': 8, + 'FAIL_ENTRY': 9, + 'INTR': 10, + 'SET_TPR': 11, + 'TPR_ACCESS': 12, + 'S390_SIEIC': 13, + 'S390_RESET': 14, + 'DCR': 15, + 'NMI': 16, + 'INTERNAL_ERROR': 17, + 'OSI': 18, + 'PAPR_HCALL': 19, + 'S390_UCONTROL': 20, + 'WATCHDOG': 21, + 'S390_TSCH': 22, + 'EPR': 23, + 'SYSTEM_EVENT': 24, } -x86_exit_reasons = { - 'vmx': vmx_exit_reasons, - 'svm': svm_exit_reasons, +IOCTL_NUMBERS = { + 'SET_FILTER': 0x40082406, + 'ENABLE': 0x00002400, + 'DISABLE': 0x00002401, + 'RESET': 0x00002403, } -sc_perf_evt_open = None -exit_reasons = None +class Arch(object): + """Class that encapsulates global architecture specific data like + syscall and ioctl numbers. + + """ + @staticmethod + def get_arch(): + machine = os.uname()[4] + + if machine.startswith('ppc'): + return ArchPPC() + elif machine.startswith('aarch64'): + return ArchA64() + elif machine.startswith('s390'): + return ArchS390() + else: + # X86_64 + for line in open('/proc/cpuinfo'): + if not line.startswith('flags'): + continue + + flags = line.split() + if 'vmx' in flags: + return ArchX86(VMX_EXIT_REASONS) + if 'svm' in flags: + return ArchX86(SVM_EXIT_REASONS) + return + +class ArchX86(Arch): + def __init__(self, exit_reasons): + self.sc_perf_evt_open = 298 + self.ioctl_numbers = IOCTL_NUMBERS + self.exit_reasons = exit_reasons + +class ArchPPC(Arch): + def __init__(self): + self.sc_perf_evt_open = 319 + self.ioctl_numbers = IOCTL_NUMBERS + self.ioctl_numbers['ENABLE'] = 0x20002400 + self.ioctl_numbers['DISABLE'] = 0x20002401 -ioctl_numbers = { - 'SET_FILTER' : 0x40082406, - 'ENABLE' : 0x00002400, - 'DISABLE' : 0x00002401, - 'RESET' : 0x00002403, -} + # PPC comes in 32 and 64 bit and some generated ioctl + # numbers depend on the wordsize. + char_ptr_size = ctypes.sizeof(ctypes.c_char_p) + self.ioctl_numbers['SET_FILTER'] = 0x80002406 | char_ptr_size << 16 + +class ArchA64(Arch): + def __init__(self): + self.sc_perf_evt_open = 241 + self.ioctl_numbers = IOCTL_NUMBERS + self.exit_reasons = AARCH64_EXIT_REASONS + +class ArchS390(Arch): + def __init__(self): + self.sc_perf_evt_open = 331 + self.ioctl_numbers = IOCTL_NUMBERS + self.exit_reasons = None + +ARCH = Arch.get_arch() + + +def walkdir(path): + """Returns os.walk() data for specified directory. + + As it is only a wrapper it returns the same 3-tuple of (dirpath, + dirnames, filenames). + """ + return next(os.walk(path)) + + +def parse_int_list(list_string): + """Returns an int list from a string of comma separated integers and + integer ranges.""" + integers = [] + members = list_string.split(',') -def x86_init(flag): - globals().update({ - 'sc_perf_evt_open' : 298, - 'exit_reasons' : x86_exit_reasons[flag], - }) - -def s390_init(): - globals().update({ - 'sc_perf_evt_open' : 331 - }) - -def ppc_init(): - globals().update({ - 'sc_perf_evt_open' : 319, - 'ioctl_numbers' : { - 'SET_FILTER' : 0x80002406 | (ctypes.sizeof(ctypes.c_char_p) << 16), - 'ENABLE' : 0x20002400, - 'DISABLE' : 0x20002401, - } - }) - -def aarch64_init(): - globals().update({ - 'sc_perf_evt_open' : 241, - 'exit_reasons' : aarch64_exit_reasons, - }) - -def detect_platform(): - if os.uname()[4].startswith('ppc'): - ppc_init() - return - elif os.uname()[4].startswith('aarch64'): - aarch64_init() - return - - for line in file('/proc/cpuinfo').readlines(): - if line.startswith('flags'): - for flag in line.split(): - if flag in x86_exit_reasons: - x86_init(flag) - return - elif line.startswith('vendor_id'): - for flag in line.split(): - if flag == 'IBM/S390': - s390_init() - return - -detect_platform() - -def invert(d): - return dict((x[1], x[0]) for x in d.iteritems()) - -filters = {} -filters['kvm_userspace_exit'] = ('reason', invert(userspace_exit_reasons)) -if exit_reasons: - filters['kvm_exit'] = ('exit_reason', invert(exit_reasons)) - -import struct, array - -libc = ctypes.CDLL('libc.so.6') + for member in members: + if '-' not in member: + integers.append(int(member)) + else: + int_range = member.split('-') + integers.extend(range(int(int_range[0]), + int(int_range[1]) + 1)) + + return integers + + +def get_online_cpus(): + with open('/sys/devices/system/cpu/online') as cpu_list: + cpu_string = cpu_list.readline() + return parse_int_list(cpu_string) + + +def get_filters(): + filters = {} + filters['kvm_userspace_exit'] = ('reason', USERSPACE_EXIT_REASONS) + if ARCH.exit_reasons: + filters['kvm_exit'] = ('exit_reason', ARCH.exit_reasons) + return filters + +libc = ctypes.CDLL('libc.so.6', use_errno=True) syscall = libc.syscall -get_errno = libc.__errno_location -get_errno.restype = POINTER(c_int) class perf_event_attr(ctypes.Structure): _fields_ = [('type', ctypes.c_uint32), @@ -305,262 +332,350 @@ class perf_event_attr(ctypes.Structure): ('bp_addr', ctypes.c_uint64), ('bp_len', ctypes.c_uint64), ] -def _perf_event_open(attr, pid, cpu, group_fd, flags): - return syscall(sc_perf_evt_open, ctypes.pointer(attr), ctypes.c_int(pid), - ctypes.c_int(cpu), ctypes.c_int(group_fd), - ctypes.c_long(flags)) - -PERF_TYPE_HARDWARE = 0 -PERF_TYPE_SOFTWARE = 1 -PERF_TYPE_TRACEPOINT = 2 -PERF_TYPE_HW_CACHE = 3 -PERF_TYPE_RAW = 4 -PERF_TYPE_BREAKPOINT = 5 - -PERF_SAMPLE_IP = 1 << 0 -PERF_SAMPLE_TID = 1 << 1 -PERF_SAMPLE_TIME = 1 << 2 -PERF_SAMPLE_ADDR = 1 << 3 -PERF_SAMPLE_READ = 1 << 4 -PERF_SAMPLE_CALLCHAIN = 1 << 5 -PERF_SAMPLE_ID = 1 << 6 -PERF_SAMPLE_CPU = 1 << 7 -PERF_SAMPLE_PERIOD = 1 << 8 -PERF_SAMPLE_STREAM_ID = 1 << 9 -PERF_SAMPLE_RAW = 1 << 10 - -PERF_FORMAT_TOTAL_TIME_ENABLED = 1 << 0 -PERF_FORMAT_TOTAL_TIME_RUNNING = 1 << 1 -PERF_FORMAT_ID = 1 << 2 -PERF_FORMAT_GROUP = 1 << 3 -import re + def __init__(self): + super(self.__class__, self).__init__() + self.type = PERF_TYPE_TRACEPOINT + self.size = ctypes.sizeof(self) + self.read_format = PERF_FORMAT_GROUP + +def perf_event_open(attr, pid, cpu, group_fd, flags): + return syscall(ARCH.sc_perf_evt_open, ctypes.pointer(attr), + ctypes.c_int(pid), ctypes.c_int(cpu), + ctypes.c_int(group_fd), ctypes.c_long(flags)) -sys_tracing = '/sys/kernel/debug/tracing' +PERF_TYPE_TRACEPOINT = 2 +PERF_FORMAT_GROUP = 1 << 3 + +PATH_DEBUGFS_TRACING = '/sys/kernel/debug/tracing' +PATH_DEBUGFS_KVM = '/sys/kernel/debug/kvm' class Group(object): - def __init__(self, cpu): + def __init__(self): self.events = [] - self.group_leader = None - self.cpu = cpu - def add_event(self, name, event_set, tracepoint, filter = None): - self.events.append(Event(group = self, - name = name, event_set = event_set, - tracepoint = tracepoint, filter = filter)) - if len(self.events) == 1: - self.file = os.fdopen(self.events[0].fd) + + def add_event(self, event): + self.events.append(event) + def read(self): - bytes = 8 * (1 + len(self.events)) - fmt = 'xxxxxxxx' + 'q' * len(self.events) + length = 8 * (1 + len(self.events)) + read_format = 'xxxxxxxx' + 'Q' * len(self.events) return dict(zip([event.name for event in self.events], - struct.unpack(fmt, self.file.read(bytes)))) + struct.unpack(read_format, + os.read(self.events[0].fd, length)))) class Event(object): - def __init__(self, group, name, event_set, tracepoint, filter = None): + def __init__(self, name, group, trace_cpu, trace_point, trace_filter, + trace_set='kvm'): self.name = name - attr = perf_event_attr() - attr.type = PERF_TYPE_TRACEPOINT - attr.size = ctypes.sizeof(attr) - id_path = os.path.join(sys_tracing, 'events', event_set, - tracepoint, 'id') - id = int(file(id_path).read()) - attr.config = id - attr.sample_type = (PERF_SAMPLE_RAW - | PERF_SAMPLE_TIME - | PERF_SAMPLE_CPU) - attr.sample_period = 1 - attr.read_format = PERF_FORMAT_GROUP + self.fd = None + self.setup_event(group, trace_cpu, trace_point, trace_filter, + trace_set) + + def setup_event_attribute(self, trace_set, trace_point): + id_path = os.path.join(PATH_DEBUGFS_TRACING, 'events', trace_set, + trace_point, 'id') + + event_attr = perf_event_attr() + event_attr.config = int(open(id_path).read()) + return event_attr + + def setup_event(self, group, trace_cpu, trace_point, trace_filter, + trace_set): + event_attr = self.setup_event_attribute(trace_set, trace_point) + group_leader = -1 if group.events: group_leader = group.events[0].fd - fd = _perf_event_open(attr, -1, group.cpu, group_leader, 0) + + fd = perf_event_open(event_attr, -1, trace_cpu, + group_leader, 0) if fd == -1: - err = get_errno()[0] - raise Exception('perf_event_open failed, errno = ' + err.__str__()) - if filter: - import fcntl - fcntl.ioctl(fd, ioctl_numbers['SET_FILTER'], filter) + err = ctypes.get_errno() + raise OSError(err, os.strerror(err), + 'while calling sys_perf_event_open().') + + if trace_filter: + fcntl.ioctl(fd, ARCH.ioctl_numbers['SET_FILTER'], + trace_filter) + self.fd = fd + def enable(self): - import fcntl - fcntl.ioctl(self.fd, ioctl_numbers['ENABLE'], 0) + fcntl.ioctl(self.fd, ARCH.ioctl_numbers['ENABLE'], 0) + def disable(self): - import fcntl - fcntl.ioctl(self.fd, ioctl_numbers['DISABLE'], 0) + fcntl.ioctl(self.fd, ARCH.ioctl_numbers['DISABLE'], 0) + def reset(self): - import fcntl - fcntl.ioctl(self.fd, ioctl_numbers['RESET'], 0) + fcntl.ioctl(self.fd, ARCH.ioctl_numbers['RESET'], 0) class TracepointProvider(object): def __init__(self): - path = os.path.join(sys_tracing, 'events', 'kvm') - fields = [f - for f in os.listdir(path) - if os.path.isdir(os.path.join(path, f))] + self.group_leaders = [] + self.filters = get_filters() + self._fields = self.get_available_fields() + self.setup_traces() + self.fields = self._fields + + def get_available_fields(self): + path = os.path.join(PATH_DEBUGFS_TRACING, 'events', 'kvm') + fields = walkdir(path)[1] extra = [] - for f in fields: - if f in filters: - subfield, values = filters[f] - for name, number in values.iteritems(): - extra.append(f + '(' + name + ')') + for field in fields: + if field in self.filters: + filter_name_, filter_dicts = self.filters[field] + for name in filter_dicts: + extra.append(field + '(' + name + ')') fields += extra - self._setup(fields) - self.select(fields) - def fields(self): - return self._fields + return fields + + def setup_traces(self): + cpus = get_online_cpus() + + # The constant is needed as a buffer for python libs, std + # streams and other files that the script opens. + newlim = len(cpus) * len(self._fields) + 50 + try: + softlim_, hardlim = resource.getrlimit(resource.RLIMIT_NOFILE) + + if hardlim < newlim: + # Now we need CAP_SYS_RESOURCE, to increase the hard limit. + resource.setrlimit(resource.RLIMIT_NOFILE, (newlim, newlim)) + else: + # Raising the soft limit is sufficient. + resource.setrlimit(resource.RLIMIT_NOFILE, (newlim, hardlim)) + + except ValueError: + sys.exit("NOFILE rlimit could not be raised to {0}".format(newlim)) - def _online_cpus(self): - l = [] - pattern = r'cpu([0-9]+)' - basedir = '/sys/devices/system/cpu' - for entry in os.listdir(basedir): - match = re.match(pattern, entry) - if not match: - continue - path = os.path.join(basedir, entry, 'online') - if os.path.exists(path) and open(path).read().strip() != '1': - continue - l.append(int(match.group(1))) - return l - - def _setup(self, _fields): - self._fields = _fields - cpus = self._online_cpus() - import resource - nfiles = len(cpus) * 1000 - resource.setrlimit(resource.RLIMIT_NOFILE, (nfiles, nfiles)) - events = [] - self.group_leaders = [] for cpu in cpus: - group = Group(cpu) - for name in _fields: + group = Group() + for name in self._fields: tracepoint = name - filter = None - m = re.match(r'(.*)\((.*)\)', name) - if m: - tracepoint, sub = m.groups() - filter = '%s==%d\0' % (filters[tracepoint][0], - filters[tracepoint][1][sub]) - event = group.add_event(name, event_set = 'kvm', - tracepoint = tracepoint, - filter = filter) + tracefilter = None + match = re.match(r'(.*)\((.*)\)', name) + if match: + tracepoint, sub = match.groups() + tracefilter = ('%s==%d\0' % + (self.filters[tracepoint][0], + self.filters[tracepoint][1][sub])) + + group.add_event(Event(name=name, + group=group, + trace_cpu=cpu, + trace_point=tracepoint, + trace_filter=tracefilter)) self.group_leaders.append(group) - def select(self, fields): + + def available_fields(self): + return self.get_available_fields() + + @property + def fields(self): + return self._fields + + @fields.setter + def fields(self, fields): + self._fields = fields for group in self.group_leaders: - for event in group.events: + for index, event in enumerate(group.events): if event.name in fields: event.reset() event.enable() else: - event.disable() + # Do not disable the group leader. + # It would disable all of its events. + if index != 0: + event.disable() + def read(self): - from collections import defaultdict ret = defaultdict(int) for group in self.group_leaders: for name, val in group.read().iteritems(): - ret[name] += val + if name in self._fields: + ret[name] += val return ret -class Stats: - def __init__(self, providers, fields = None): +class DebugfsProvider(object): + def __init__(self): + self._fields = self.get_available_fields() + + def get_available_fields(self): + return walkdir(PATH_DEBUGFS_KVM)[2] + + @property + def fields(self): + return self._fields + + @fields.setter + def fields(self, fields): + self._fields = fields + + def read(self): + def val(key): + return int(file(PATH_DEBUGFS_KVM + '/' + key).read()) + return dict([(key, val(key)) for key in self._fields]) + +class Stats(object): + def __init__(self, providers, fields=None): self.providers = providers - self.fields_filter = fields - self._update() - def _update(self): + self._fields_filter = fields + self.values = {} + self.update_provider_filters() + + def update_provider_filters(self): def wanted(key): - import re - if not self.fields_filter: + if not self._fields_filter: return True - return re.match(self.fields_filter, key) is not None - self.values = dict() - for d in providers: - provider_fields = [key for key in d.fields() if wanted(key)] - for key in provider_fields: - self.values[key] = None - d.select(provider_fields) - def set_fields_filter(self, fields_filter): - self.fields_filter = fields_filter - self._update() + return re.match(self._fields_filter, key) is not None + + # As we reset the counters when updating the fields we can + # also clear the cache of old values. + self.values = {} + for provider in self.providers: + provider_fields = [key for key in provider.get_available_fields() + if wanted(key)] + provider.fields = provider_fields + + @property + def fields_filter(self): + return self._fields_filter + + @fields_filter.setter + def fields_filter(self, fields_filter): + self._fields_filter = fields_filter + self.update_provider_filters() + def get(self): - for d in providers: - new = d.read() - for key in d.fields(): + for provider in self.providers: + new = provider.read() + for key in provider.fields: oldval = self.values.get(key, (0, 0)) - newval = new[key] + newval = new.get(key, 0) newdelta = None if oldval is not None: newdelta = newval - oldval[0] self.values[key] = (newval, newdelta) return self.values -if not os.access('/sys/kernel/debug', os.F_OK): - print 'Please enable CONFIG_DEBUG_FS in your kernel' - sys.exit(1) -if not os.access('/sys/kernel/debug/kvm', os.F_OK): - print "Please mount debugfs ('mount -t debugfs debugfs /sys/kernel/debug')" - print "and ensure the kvm modules are loaded" - sys.exit(1) - -label_width = 40 -number_width = 10 - -def tui(screen, stats): - curses.use_default_colors() - curses.noecho() - drilldown = False - fields_filter = stats.fields_filter - def update_drilldown(): - if not fields_filter: - if drilldown: - stats.set_fields_filter(None) - else: - stats.set_fields_filter(r'^[^\(]*$') - update_drilldown() - def refresh(sleeptime): - screen.erase() - screen.addstr(0, 0, 'kvm statistics') - screen.addstr(2, 1, 'Event') - screen.addstr(2, 1 + label_width + number_width - len('Total'), 'Total') - screen.addstr(2, 1 + label_width + number_width + 8 - len('Current'), 'Current') +LABEL_WIDTH = 40 +NUMBER_WIDTH = 10 + +class Tui(object): + def __init__(self, stats): + self.stats = stats + self.screen = None + self.drilldown = False + self.update_drilldown() + + def __enter__(self): + """Initialises curses for later use. Based on curses.wrapper + implementation from the Python standard library.""" + self.screen = curses.initscr() + curses.noecho() + curses.cbreak() + + # The try/catch works around a minor bit of + # over-conscientiousness in the curses module, the error + # return from C start_color() is ignorable. + try: + curses.start_color() + except: + pass + + curses.use_default_colors() + return self + + def __exit__(self, *exception): + """Resets the terminal to its normal state. Based on curses.wrappre + implementation from the Python standard library.""" + if self.screen: + self.screen.keypad(0) + curses.echo() + curses.nocbreak() + curses.endwin() + + def update_drilldown(self): + if not self.stats.fields_filter: + self.stats.fields_filter = r'^[^\(]*$' + + elif self.stats.fields_filter == r'^[^\(]*$': + self.stats.fields_filter = None + + def refresh(self, sleeptime): + self.screen.erase() + self.screen.addstr(0, 0, 'kvm statistics - summary', curses.A_BOLD) + self.screen.addstr(2, 1, 'Event') + self.screen.addstr(2, 1 + LABEL_WIDTH + NUMBER_WIDTH - + len('Total'), 'Total') + self.screen.addstr(2, 1 + LABEL_WIDTH + NUMBER_WIDTH + 8 - + len('Current'), 'Current') row = 3 - s = stats.get() + stats = self.stats.get() def sortkey(x): - if s[x][1]: - return (-s[x][1], -s[x][0]) + if stats[x][1]: + return (-stats[x][1], -stats[x][0]) else: - return (0, -s[x][0]) - for key in sorted(s.keys(), key = sortkey): - if row >= screen.getmaxyx()[0]: + return (0, -stats[x][0]) + for key in sorted(stats.keys(), key=sortkey): + + if row >= self.screen.getmaxyx()[0]: break - values = s[key] + values = stats[key] if not values[0] and not values[1]: break col = 1 - screen.addstr(row, col, key) - col += label_width - screen.addstr(row, col, '%10d' % (values[0],)) - col += number_width + self.screen.addstr(row, col, key) + col += LABEL_WIDTH + self.screen.addstr(row, col, '%10d' % (values[0],)) + col += NUMBER_WIDTH if values[1] is not None: - screen.addstr(row, col, '%8d' % (values[1] / sleeptime,)) + self.screen.addstr(row, col, '%8d' % (values[1] / sleeptime,)) row += 1 - screen.refresh() + self.screen.refresh() + + def show_filter_selection(self): + while True: + self.screen.erase() + self.screen.addstr(0, 0, + "Show statistics for events matching a regex.", + curses.A_BOLD) + self.screen.addstr(2, 0, + "Current regex: {0}" + .format(self.stats.fields_filter)) + self.screen.addstr(3, 0, "New regex: ") + curses.echo() + regex = self.screen.getstr() + curses.noecho() + if len(regex) == 0: + return + try: + re.compile(regex) + self.stats.fields_filter = regex + return + except re.error: + continue - sleeptime = 0.25 - while True: - refresh(sleeptime) - curses.halfdelay(int(sleeptime * 10)) - sleeptime = 3 - try: - c = screen.getkey() - if c == 'x': - drilldown = not drilldown - update_drilldown() - if c == 'q': + def show_stats(self): + sleeptime = 0.25 + while True: + self.refresh(sleeptime) + curses.halfdelay(int(sleeptime * 10)) + sleeptime = 3 + try: + char = self.screen.getkey() + if char == 'x': + self.drilldown = not self.drilldown + self.update_drilldown() + if char == 'q': + break + if char == 'f': + self.show_filter_selection() + except KeyboardInterrupt: break - except KeyboardInterrupt: - break - except curses.error: - continue + except curses.error: + continue def batch(stats): s = stats.get() @@ -568,13 +683,13 @@ def batch(stats): s = stats.get() for key in sorted(s.keys()): values = s[key] - print '%-22s%10d%10d' % (key, values[0], values[1]) + print '%-42s%10d%10d' % (key, values[0], values[1]) def log(stats): keys = sorted(stats.get().iterkeys()) def banner(): for k in keys: - print '%10s' % k[0:9], + print '%s' % k, print def statline(): s = stats.get() @@ -590,57 +705,121 @@ def log(stats): statline() line += 1 -options = optparse.OptionParser() -options.add_option('-1', '--once', '--batch', - action = 'store_true', - default = False, - dest = 'once', - help = 'run in batch mode for one second', - ) -options.add_option('-l', '--log', - action = 'store_true', - default = False, - dest = 'log', - help = 'run in logging mode (like vmstat)', - ) -options.add_option('-t', '--tracepoints', - action = 'store_true', - default = False, - dest = 'tracepoints', - help = 'retrieve statistics from tracepoints', - ) -options.add_option('-d', '--debugfs', - action = 'store_true', - default = False, - dest = 'debugfs', - help = 'retrieve statistics from debugfs', - ) -options.add_option('-f', '--fields', - action = 'store', - default = None, - dest = 'fields', - help = 'fields to display (regex)', - ) -(options, args) = options.parse_args(sys.argv) - -providers = [] -if options.tracepoints: - providers.append(TracepointProvider()) -if options.debugfs: - providers.append(DebugfsProvider()) - -if len(providers) == 0: - try: - providers = [TracepointProvider()] - except: - providers = [DebugfsProvider()] - -stats = Stats(providers, fields = options.fields) - -if options.log: - log(stats) -elif not options.once: - import curses.wrapper - curses.wrapper(tui, stats) -else: - batch(stats) +def get_options(): + description_text = """ +This script displays various statistics about VMs running under KVM. +The statistics are gathered from the KVM debugfs entries and / or the +currently available perf traces. + +The monitoring takes additional cpu cycles and might affect the VM's +performance. + +Requirements: +- Access to: + /sys/kernel/debug/kvm + /sys/kernel/debug/trace/events/* + /proc/pid/task +- /proc/sys/kernel/perf_event_paranoid < 1 if user has no + CAP_SYS_ADMIN and perf events are used. +- CAP_SYS_RESOURCE if the hard limit is not high enough to allow + the large number of files that are possibly opened. +""" + + class PlainHelpFormatter(optparse.IndentedHelpFormatter): + def format_description(self, description): + if description: + return description + "\n" + else: + return "" + + optparser = optparse.OptionParser(description=description_text, + formatter=PlainHelpFormatter()) + optparser.add_option('-1', '--once', '--batch', + action='store_true', + default=False, + dest='once', + help='run in batch mode for one second', + ) + optparser.add_option('-l', '--log', + action='store_true', + default=False, + dest='log', + help='run in logging mode (like vmstat)', + ) + optparser.add_option('-t', '--tracepoints', + action='store_true', + default=False, + dest='tracepoints', + help='retrieve statistics from tracepoints', + ) + optparser.add_option('-d', '--debugfs', + action='store_true', + default=False, + dest='debugfs', + help='retrieve statistics from debugfs', + ) + optparser.add_option('-f', '--fields', + action='store', + default=None, + dest='fields', + help='fields to display (regex)', + ) + (options, _) = optparser.parse_args(sys.argv) + return options + +def get_providers(options): + providers = [] + + if options.tracepoints: + providers.append(TracepointProvider()) + if options.debugfs: + providers.append(DebugfsProvider()) + if len(providers) == 0: + providers.append(TracepointProvider()) + + return providers + +def check_access(options): + if not os.path.exists('/sys/kernel/debug'): + sys.stderr.write('Please enable CONFIG_DEBUG_FS in your kernel.') + sys.exit(1) + + if not os.path.exists(PATH_DEBUGFS_KVM): + sys.stderr.write("Please make sure, that debugfs is mounted and " + "readable by the current user:\n" + "('mount -t debugfs debugfs /sys/kernel/debug')\n" + "Also ensure, that the kvm modules are loaded.\n") + sys.exit(1) + + if not os.path.exists(PATH_DEBUGFS_TRACING) and (options.tracepoints + or not options.debugfs): + sys.stderr.write("Please enable CONFIG_TRACING in your kernel " + "when using the option -t (default).\n" + "If it is enabled, make {0} readable by the " + "current user.\n" + .format(PATH_DEBUGFS_TRACING)) + if options.tracepoints: + sys.exit(1) + + sys.stderr.write("Falling back to debugfs statistics!\n") + options.debugfs = True + sleep(5) + + return options + +def main(): + options = get_options() + options = check_access(options) + providers = get_providers(options) + stats = Stats(providers, fields=options.fields) + + if options.log: + log(stats) + elif not options.once: + with Tui(stats) as tui: + tui.show_stats() + else: + batch(stats) + +if __name__ == "__main__": + main() diff --git a/qemu/scripts/ordereddict.py b/qemu/scripts/ordereddict.py index 7242b5060..2d1d81370 100644 --- a/qemu/scripts/ordereddict.py +++ b/qemu/scripts/ordereddict.py @@ -22,6 +22,7 @@ from UserDict import DictMixin + class OrderedDict(dict, DictMixin): def __init__(self, *args, **kwds): @@ -117,7 +118,7 @@ class OrderedDict(dict, DictMixin): if isinstance(other, OrderedDict): if len(self) != len(other): return False - for p, q in zip(self.items(), other.items()): + for p, q in zip(self.items(), other.items()): if p != q: return False return True diff --git a/qemu/scripts/qapi-commands.py b/qemu/scripts/qapi-commands.py index ca22acc1d..b570069fa 100644 --- a/qemu/scripts/qapi-commands.py +++ b/qemu/scripts/qapi-commands.py @@ -2,7 +2,7 @@ # QAPI command marshaller generator # # Copyright IBM, Corp. 2011 -# Copyright (C) 2014-2015 Red Hat, Inc. +# Copyright (C) 2014-2016 Red Hat, Inc. # # Authors: # Anthony Liguori <aliguori@us.ibm.com> @@ -12,274 +12,217 @@ # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. -from ordereddict import OrderedDict from qapi import * import re -def generate_command_decl(name, args, ret_type): - arglist="" - for argname, argtype, optional in parse_args(args): - argtype = c_type(argtype, is_param=True) - if optional: - arglist += "bool has_%s, " % c_name(argname) - arglist += "%s %s, " % (argtype, c_name(argname)) + +def gen_command_decl(name, arg_type, ret_type): return mcgen(''' -%(ret_type)s qmp_%(name)s(%(args)sError **errp); +%(c_type)s qmp_%(c_name)s(%(params)s); ''', - ret_type=c_type(ret_type), name=c_name(name), - args=arglist).strip() - -def gen_err_check(errvar): - if errvar: - return mcgen(''' -if (local_err) { - goto out; -} -''') - return '' + c_type=(ret_type and ret_type.c_type()) or 'void', + c_name=c_name(name), + params=gen_params(arg_type, 'Error **errp')) -def gen_sync_call(name, args, ret_type, indent=0): - ret = "" - arglist="" - retval="" - if ret_type: - retval = "retval = " - for argname, argtype, optional in parse_args(args): - if optional: - arglist += "has_%s, " % c_name(argname) - arglist += "%s, " % (c_name(argname)) - push_indent(indent) - ret = mcgen(''' -%(retval)sqmp_%(name)s(%(args)s&local_err); -''', - name=c_name(name), args=arglist, retval=retval).rstrip() - if ret_type: - ret += "\n" + gen_err_check('local_err') - ret += "\n" + mcgen('''' -%(marshal_output_call)s -''', - marshal_output_call=gen_marshal_output_call(name, ret_type)).rstrip() - pop_indent(indent) - return ret.rstrip() +def gen_call(name, arg_type, ret_type): + ret = '' + argstr = '' + if arg_type: + assert not arg_type.variants + for memb in arg_type.members: + if memb.optional: + argstr += 'arg.has_%s, ' % c_name(memb.name) + argstr += 'arg.%s, ' % c_name(memb.name) -def gen_marshal_output_call(name, ret_type): - if not ret_type: - return "" - return "qmp_marshal_output_%s(retval, ret, &local_err);" % c_name(name) + lhs = '' + if ret_type: + lhs = 'retval = ' -def gen_visitor_input_containers_decl(args, obj): - ret = "" + ret = mcgen(''' - push_indent() - if len(args) > 0: - ret += mcgen(''' -QmpInputVisitor *mi = qmp_input_visitor_new_strict(%(obj)s); -QapiDeallocVisitor *md; -Visitor *v; -''', - obj=obj) - pop_indent() - - return ret.rstrip() - -def gen_visitor_input_vars_decl(args): - ret = "" - push_indent() - for argname, argtype, optional in parse_args(args): - if optional: - ret += mcgen(''' -bool has_%(argname)s = false; -''', - argname=c_name(argname)) - if is_c_ptr(argtype): - ret += mcgen(''' -%(argtype)s %(argname)s = NULL; + %(lhs)sqmp_%(c_name)s(%(args)s&err); ''', - argname=c_name(argname), argtype=c_type(argtype)) - else: - ret += mcgen(''' -%(argtype)s %(argname)s = {0}; -''', - argname=c_name(argname), argtype=c_type(argtype)) - - pop_indent() - return ret.rstrip() - -def gen_visitor_input_block(args, dealloc=False): - ret = "" - errparg = '&local_err' - errarg = 'local_err' - - if len(args) == 0: - return ret - - push_indent() - - if dealloc: - errparg = 'NULL' - errarg = None; - ret += mcgen(''' -qmp_input_visitor_cleanup(mi); -md = qapi_dealloc_visitor_new(); -v = qapi_dealloc_get_visitor(md); -''') - else: + c_name=c_name(name), args=argstr, lhs=lhs) + if ret_type: + ret += gen_err_check() ret += mcgen(''' -v = qmp_input_get_visitor(mi); -''') - for argname, argtype, optional in parse_args(args): - if optional: - ret += mcgen(''' -visit_optional(v, &has_%(c_name)s, "%(name)s", %(errp)s); + qmp_marshal_output_%(c_name)s(retval, ret, &err); ''', - c_name=c_name(argname), name=argname, errp=errparg) - ret += gen_err_check(errarg) - ret += mcgen(''' -if (has_%(c_name)s) { -''', - c_name=c_name(argname)) - push_indent() - ret += mcgen(''' -visit_type_%(visitor)s(v, &%(c_name)s, "%(name)s", %(errp)s); -''', - c_name=c_name(argname), name=argname, argtype=argtype, - visitor=type_name(argtype), errp=errparg) - ret += gen_err_check(errarg) - if optional: - pop_indent() - ret += mcgen(''' -} -''') + c_name=ret_type.c_name()) + return ret - if dealloc: - ret += mcgen(''' -qapi_dealloc_visitor_cleanup(md); -''') - pop_indent() - return ret.rstrip() -def gen_marshal_output(name, args, ret_type, middle_mode): - if not ret_type: - return "" +def gen_marshal_output(ret_type): + return mcgen(''' - ret = mcgen(''' -static void qmp_marshal_output_%(c_name)s(%(c_ret_type)s ret_in, QObject **ret_out, Error **errp) +static void qmp_marshal_output_%(c_name)s(%(c_type)s ret_in, QObject **ret_out, Error **errp) { - Error *local_err = NULL; - QmpOutputVisitor *mo = qmp_output_visitor_new(); - QapiDeallocVisitor *md; + Error *err = NULL; + QmpOutputVisitor *qov = qmp_output_visitor_new(); + QapiDeallocVisitor *qdv; Visitor *v; - v = qmp_output_get_visitor(mo); - visit_type_%(visitor)s(v, &ret_in, "unused", &local_err); - if (local_err) { + v = qmp_output_get_visitor(qov); + visit_type_%(c_name)s(v, "unused", &ret_in, &err); + if (err) { goto out; } - *ret_out = qmp_output_get_qobject(mo); + *ret_out = qmp_output_get_qobject(qov); out: - error_propagate(errp, local_err); - qmp_output_visitor_cleanup(mo); - md = qapi_dealloc_visitor_new(); - v = qapi_dealloc_get_visitor(md); - visit_type_%(visitor)s(v, &ret_in, "unused", NULL); - qapi_dealloc_visitor_cleanup(md); + error_propagate(errp, err); + qmp_output_visitor_cleanup(qov); + qdv = qapi_dealloc_visitor_new(); + v = qapi_dealloc_get_visitor(qdv); + visit_type_%(c_name)s(v, "unused", &ret_in, NULL); + qapi_dealloc_visitor_cleanup(qdv); } ''', - c_ret_type=c_type(ret_type), c_name=c_name(name), - visitor=type_name(ret_type)) + c_type=ret_type.c_type(), c_name=ret_type.c_name()) - return ret -def gen_marshal_input_decl(name, args, ret_type, middle_mode): - ret = 'void qmp_marshal_input_%s(QDict *args, QObject **ret, Error **errp)' % c_name(name) +def gen_marshal_proto(name): + ret = 'void qmp_marshal_%s(QDict *args, QObject **ret, Error **errp)' % c_name(name) if not middle_mode: - ret = "static " + ret + ret = 'static ' + ret return ret -def gen_marshal_input(name, args, ret_type, middle_mode): - hdr = gen_marshal_input_decl(name, args, ret_type, middle_mode) +def gen_marshal_decl(name): + return mcgen(''' +%(proto)s; +''', + proto=gen_marshal_proto(name)) + + +def gen_marshal(name, arg_type, ret_type): ret = mcgen(''' -%(header)s + +%(proto)s { - Error *local_err = NULL; + Error *err = NULL; ''', - header=hdr) + proto=gen_marshal_proto(name)) if ret_type: - if is_c_ptr(ret_type): - retval = " %s retval = NULL;" % c_type(ret_type) - else: - retval = " %s retval;" % c_type(ret_type) ret += mcgen(''' -%(retval)s + %(c_type)s retval; ''', - retval=retval) + c_type=ret_type.c_type()) - if len(args) > 0: + if arg_type and arg_type.members: ret += mcgen(''' -%(visitor_input_containers_decl)s -%(visitor_input_vars_decl)s - -%(visitor_input_block)s + QmpInputVisitor *qiv = qmp_input_visitor_new_strict(QOBJECT(args)); + QapiDeallocVisitor *qdv; + Visitor *v; + %(c_name)s arg = {0}; + v = qmp_input_get_visitor(qiv); + visit_type_%(c_name)s_members(v, &arg, &err); + if (err) { + goto out; + } ''', - visitor_input_containers_decl=gen_visitor_input_containers_decl(args, "QOBJECT(args)"), - visitor_input_vars_decl=gen_visitor_input_vars_decl(args), - visitor_input_block=gen_visitor_input_block(args)) + c_name=arg_type.c_name()) + else: ret += mcgen(''' (void)args; ''') - ret += mcgen(''' -%(sync_call)s -''', - sync_call=gen_sync_call(name, args, ret_type, indent=4)) - if re.search('^ *goto out\\;', ret, re.MULTILINE): + ret += gen_call(name, arg_type, ret_type) + + # 'goto out' produced above for arg_type, and by gen_call() for ret_type + if (arg_type and arg_type.members) or ret_type: ret += mcgen(''' out: ''') ret += mcgen(''' - error_propagate(errp, local_err); -%(visitor_input_block_cleanup)s -} + error_propagate(errp, err); +''') + if arg_type and arg_type.members: + ret += mcgen(''' + qmp_input_visitor_cleanup(qiv); + qdv = qapi_dealloc_visitor_new(); + v = qapi_dealloc_get_visitor(qdv); + visit_type_%(c_name)s_members(v, &arg, NULL); + qapi_dealloc_visitor_cleanup(qdv); ''', - visitor_input_block_cleanup=gen_visitor_input_block(args, - dealloc=True)) + c_name=arg_type.c_name()) + + ret += mcgen(''' +} +''') return ret -def gen_registry(commands): - registry="" - push_indent() - for cmd in commands: - options = 'QCO_NO_OPTIONS' - if not cmd.get('success-response', True): - options = 'QCO_NO_SUCCESS_RESP' - registry += mcgen(''' -qmp_register_command("%(name)s", qmp_marshal_input_%(c_name)s, %(opts)s); +def gen_register_command(name, success_response): + options = 'QCO_NO_OPTIONS' + if not success_response: + options = 'QCO_NO_SUCCESS_RESP' + + ret = mcgen(''' + qmp_register_command("%(name)s", qmp_marshal_%(c_name)s, %(opts)s); ''', - name=cmd['command'], c_name=c_name(cmd['command']), - opts=options) - pop_indent() + name=name, c_name=c_name(name), + opts=options) + return ret + + +def gen_registry(registry): ret = mcgen(''' + static void qmp_init_marshal(void) { -%(registry)s +''') + ret += registry + ret += mcgen(''' } qapi_init(qmp_init_marshal); -''', - registry=registry.rstrip()) +''') return ret + +class QAPISchemaGenCommandVisitor(QAPISchemaVisitor): + def __init__(self): + self.decl = None + self.defn = None + self._regy = None + self._visited_ret_types = None + + def visit_begin(self, schema): + self.decl = '' + self.defn = '' + self._regy = '' + self._visited_ret_types = set() + + def visit_end(self): + if not middle_mode: + self.defn += gen_registry(self._regy) + self._regy = None + self._visited_ret_types = None + + def visit_command(self, name, info, arg_type, ret_type, + gen, success_response): + if not gen: + return + self.decl += gen_command_decl(name, arg_type, ret_type) + if ret_type and ret_type not in self._visited_ret_types: + self._visited_ret_types.add(ret_type) + self.defn += gen_marshal_output(ret_type) + if middle_mode: + self.decl += gen_marshal_decl(name) + self.defn += gen_marshal(name, arg_type, ret_type) + if not middle_mode: + self._regy += gen_register_command(name, success_response) + + middle_mode = False (input_file, output_dir, do_c, do_h, prefix, opts) = \ @@ -289,10 +232,6 @@ for o, a in opts: if o in ("-m", "--middle"): middle_mode = True -exprs = parse_schema(input_file) -commands = filter(lambda expr: expr.has_key('command'), exprs) -commands = filter(lambda expr: not expr.has_key('gen'), commands) - c_comment = ''' /* * schema-defined QMP->QAPI command dispatch @@ -327,6 +266,7 @@ h_comment = ''' c_comment, h_comment) fdef.write(mcgen(''' +#include "qemu/osdep.h" #include "qemu-common.h" #include "qemu/module.h" #include "qapi/qmp/types.h" @@ -340,7 +280,7 @@ fdef.write(mcgen(''' #include "%(prefix)sqmp-commands.h" ''', - prefix=prefix)) + prefix=prefix)) fdecl.write(mcgen(''' #include "%(prefix)sqapi-types.h" @@ -348,29 +288,12 @@ fdecl.write(mcgen(''' #include "qapi/error.h" ''', - prefix=prefix)) - -for cmd in commands: - arglist = [] - ret_type = None - if cmd.has_key('data'): - arglist = cmd['data'] - if cmd.has_key('returns'): - ret_type = cmd['returns'] - ret = generate_command_decl(cmd['command'], arglist, ret_type) + "\n" - fdecl.write(ret) - if ret_type: - ret = gen_marshal_output(cmd['command'], arglist, ret_type, middle_mode) + "\n" - fdef.write(ret) - - if middle_mode: - fdecl.write('%s;\n' % gen_marshal_input_decl(cmd['command'], arglist, ret_type, middle_mode)) - - ret = gen_marshal_input(cmd['command'], arglist, ret_type, middle_mode) + "\n" - fdef.write(ret) + prefix=prefix)) -if not middle_mode: - ret = gen_registry(commands) - fdef.write(ret) +schema = QAPISchema(input_file) +gen = QAPISchemaGenCommandVisitor() +schema.visit(gen) +fdef.write(gen.defn) +fdecl.write(gen.decl) close_output(fdef, fdecl) diff --git a/qemu/scripts/qapi-event.py b/qemu/scripts/qapi-event.py index 56bc602a6..9b5c5b535 100644 --- a/qemu/scripts/qapi-event.py +++ b/qemu/scripts/qapi-event.py @@ -2,215 +2,151 @@ # QAPI event generator # # Copyright (c) 2014 Wenchao Xia +# Copyright (c) 2015-2016 Red Hat Inc. # # Authors: # Wenchao Xia <wenchaoqemu@gmail.com> +# Markus Armbruster <armbru@redhat.com> # # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. -from ordereddict import OrderedDict from qapi import * -def _generate_event_api_name(event_name, params): - api_name = "void qapi_event_send_%s(" % c_name(event_name).lower(); - l = len(api_name) - if params: - for argname, argentry, optional in parse_args(params): - if optional: - api_name += "bool has_%s,\n" % c_name(argname) - api_name += "".ljust(l) +def gen_event_send_proto(name, arg_type): + return 'void qapi_event_send_%(c_name)s(%(param)s)' % { + 'c_name': c_name(name.lower()), + 'param': gen_params(arg_type, 'Error **errp')} - api_name += "%s %s,\n" % (c_type(argentry, is_param=True), - c_name(argname)) - api_name += "".ljust(l) - - api_name += "Error **errp)" - return api_name; +def gen_event_send_decl(name, arg_type): + return mcgen(''' -# Following are the core functions that generate C APIs to emit event. +%(proto)s; +''', + proto=gen_event_send_proto(name, arg_type)) -def generate_event_declaration(api_name): - return mcgen(''' -%(api_name)s; +# Declare and initialize an object 'qapi' using parameters from gen_params() +def gen_param_var(typ): + assert not typ.variants + ret = mcgen(''' + %(c_name)s param = { ''', - api_name = api_name) + c_name=typ.c_name()) + sep = ' ' + for memb in typ.members: + ret += sep + sep = ', ' + if memb.optional: + ret += 'has_' + c_name(memb.name) + sep + if memb.type.name == 'str': + # Cast away const added in gen_params() + ret += '(char *)' + ret += c_name(memb.name) + ret += mcgen(''' + + }; +''') + return ret -def generate_event_implement(api_name, event_name, params): - # step 1: declare any variables - ret = mcgen(""" -%(api_name)s +def gen_event_send(name, arg_type): + # FIXME: Our declaration of local variables (and of 'errp' in the + # parameter list) can collide with exploded members of the event's + # data type passed in as parameters. If this collision ever hits in + # practice, we can rename our local variables with a leading _ prefix, + # or split the code into a wrapper function that creates a boxed + # 'param' object then calls another to do the real work. + ret = mcgen(''' + +%(proto)s { QDict *qmp; - Error *local_err = NULL; + Error *err = NULL; QMPEventFuncEmit emit; -""", - api_name = api_name) +''', + proto=gen_event_send_proto(name, arg_type)) - if params: - ret += mcgen(""" + if arg_type and arg_type.members: + ret += mcgen(''' QmpOutputVisitor *qov; Visitor *v; - QObject *obj; +''') + ret += gen_param_var(arg_type) -""") + ret += mcgen(''' - # step 2: check emit function, create a dict - ret += mcgen(""" emit = qmp_event_get_func_emit(); if (!emit) { return; } - qmp = qmp_event_build_dict("%(event_name)s"); + qmp = qmp_event_build_dict("%(name)s"); -""", - event_name = event_name) +''', + name=name) - # step 3: visit the params if params != None - if params: - ret += mcgen(""" + if arg_type and arg_type.members: + ret += mcgen(''' qov = qmp_output_visitor_new(); - g_assert(qov); - v = qmp_output_get_visitor(qov); - g_assert(v); - /* Fake visit, as if all members are under a structure */ - visit_start_struct(v, NULL, "", "%(event_name)s", 0, &local_err); - if (local_err) { - goto clean; - } - -""", - event_name = event_name) - - for argname, argentry, optional in parse_args(params): - if optional: - ret += mcgen(""" - if (has_%(var)s) { -""", - var = c_name(argname)) - push_indent() - - if argentry == "str": - var_type = "(char **)" - else: - var_type = "" - - ret += mcgen(""" - visit_type_%(type)s(v, %(var_type)s&%(var)s, "%(name)s", &local_err); - if (local_err) { - goto clean; - } -""", - var_type = var_type, - var = c_name(argname), - type = type_name(argentry), - name = argname) - - if optional: - pop_indent() - ret += mcgen(""" + visit_start_struct(v, "%(name)s", NULL, 0, &err); + if (err) { + goto out; } -""") - - ret += mcgen(""" - - visit_end_struct(v, &local_err); - if (local_err) { - goto clean; + visit_type_%(c_name)s_members(v, ¶m, &err); + visit_end_struct(v, err ? NULL : &err); + if (err) { + goto out; } - obj = qmp_output_get_qobject(qov); - g_assert(obj != NULL); - - qdict_put_obj(qmp, "data", obj); -""") + qdict_put_obj(qmp, "data", qmp_output_get_qobject(qov)); +''', + name=name, c_name=arg_type.c_name()) - # step 4: call qmp event api - ret += mcgen(""" - emit(%(event_enum_value)s, qmp, &local_err); + ret += mcgen(''' + emit(%(c_enum)s, qmp, &err); -""", - event_enum_value = event_enum_value) +''', + c_enum=c_enum_const(event_enum_name, name)) - # step 5: clean up - if params: - ret += mcgen(""" - clean: + if arg_type and arg_type.members: + ret += mcgen(''' +out: qmp_output_visitor_cleanup(qov); -""") - ret += mcgen(""" - error_propagate(errp, local_err); +''') + ret += mcgen(''' + error_propagate(errp, err); QDECREF(qmp); } -""") - +''') return ret -# Following are the functions that generate an enum type for all defined -# events, similar to qapi-types.py. Here we already have enum name and -# values which were generated before and recorded in event_enum_*. It also -# works around the issue that "import qapi-types" can't work. - -def generate_event_enum_decl(event_enum_name, event_enum_values): - lookup_decl = mcgen(''' - -extern const char *%(event_enum_name)s_lookup[]; -''', - event_enum_name = event_enum_name) - - enum_decl = mcgen(''' -typedef enum %(event_enum_name)s -{ -''', - event_enum_name = event_enum_name) - - # append automatically generated _MAX value - enum_max_value = c_enum_const(event_enum_name, "MAX") - enum_values = event_enum_values + [ enum_max_value ] - - i = 0 - for value in enum_values: - enum_decl += mcgen(''' - %(value)s = %(i)d, -''', - value = value, - i = i) - i += 1 - - enum_decl += mcgen(''' -} %(event_enum_name)s; -''', - event_enum_name = event_enum_name) - - return lookup_decl + enum_decl +class QAPISchemaGenEventVisitor(QAPISchemaVisitor): + def __init__(self): + self.decl = None + self.defn = None + self._event_names = None -def generate_event_enum_lookup(event_enum_name, event_enum_strings): - ret = mcgen(''' + def visit_begin(self, schema): + self.decl = '' + self.defn = '' + self._event_names = [] -const char *%(event_enum_name)s_lookup[] = { -''', - event_enum_name = event_enum_name) + def visit_end(self): + self.decl += gen_enum(event_enum_name, self._event_names) + self.defn += gen_enum_lookup(event_enum_name, self._event_names) + self._event_names = None - i = 0 - for string in event_enum_strings: - ret += mcgen(''' - "%(string)s", -''', - string = string) + def visit_event(self, name, info, arg_type): + self.decl += gen_event_send_decl(name, arg_type) + self.defn += gen_event_send(name, arg_type) + self._event_names.append(name) - ret += mcgen(''' - NULL, -}; -''') - return ret (input_file, output_dir, do_c, do_h, prefix, dummy) = parse_command_line() @@ -248,6 +184,7 @@ h_comment = ''' c_comment, h_comment) fdef.write(mcgen(''' +#include "qemu/osdep.h" #include "qemu-common.h" #include "%(prefix)sqapi-event.h" #include "%(prefix)sqapi-visit.h" @@ -265,35 +202,12 @@ fdecl.write(mcgen(''' ''', prefix=prefix)) -exprs = parse_schema(input_file) - -event_enum_name = prefix.upper().replace('-', '_') + "QAPIEvent" -event_enum_values = [] -event_enum_strings = [] - -for expr in exprs: - if expr.has_key('event'): - event_name = expr['event'] - params = expr.get('data') - if params and len(params) == 0: - params = None - - api_name = _generate_event_api_name(event_name, params) - ret = generate_event_declaration(api_name) - fdecl.write(ret) - - # We need an enum value per event - event_enum_value = c_enum_const(event_enum_name, event_name) - ret = generate_event_implement(api_name, event_name, params) - fdef.write(ret) - - # Record it, and generate enum later - event_enum_values.append(event_enum_value) - event_enum_strings.append(event_name) +event_enum_name = c_name(prefix + "QAPIEvent", protect=False) -ret = generate_event_enum_decl(event_enum_name, event_enum_values) -fdecl.write(ret) -ret = generate_event_enum_lookup(event_enum_name, event_enum_strings) -fdef.write(ret) +schema = QAPISchema(input_file) +gen = QAPISchemaGenEventVisitor() +schema.visit(gen) +fdef.write(gen.defn) +fdecl.write(gen.decl) close_output(fdef, fdecl) diff --git a/qemu/scripts/qapi-introspect.py b/qemu/scripts/qapi-introspect.py new file mode 100644 index 000000000..e0f926be0 --- /dev/null +++ b/qemu/scripts/qapi-introspect.py @@ -0,0 +1,219 @@ +# +# QAPI introspection generator +# +# Copyright (C) 2015-2016 Red Hat, Inc. +# +# Authors: +# Markus Armbruster <armbru@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2. +# See the COPYING file in the top-level directory. + +from qapi import * + + +# Caveman's json.dumps() replacement (we're stuck at Python 2.4) +# TODO try to use json.dumps() once we get unstuck +def to_json(obj, level=0): + if obj is None: + ret = 'null' + elif isinstance(obj, str): + ret = '"' + obj.replace('"', r'\"') + '"' + elif isinstance(obj, list): + elts = [to_json(elt, level + 1) + for elt in obj] + ret = '[' + ', '.join(elts) + ']' + elif isinstance(obj, dict): + elts = ['"%s": %s' % (key.replace('"', r'\"'), + to_json(obj[key], level + 1)) + for key in sorted(obj.keys())] + ret = '{' + ', '.join(elts) + '}' + else: + assert False # not implemented + if level == 1: + ret = '\n' + ret + return ret + + +def to_c_string(string): + return '"' + string.replace('\\', r'\\').replace('"', r'\"') + '"' + + +class QAPISchemaGenIntrospectVisitor(QAPISchemaVisitor): + def __init__(self, unmask): + self._unmask = unmask + self.defn = None + self.decl = None + self._schema = None + self._jsons = None + self._used_types = None + self._name_map = None + + def visit_begin(self, schema): + self._schema = schema + self._jsons = [] + self._used_types = [] + self._name_map = {} + + def visit_end(self): + # visit the types that are actually used + jsons = self._jsons + self._jsons = [] + for typ in self._used_types: + typ.visit(self) + # generate C + # TODO can generate awfully long lines + jsons.extend(self._jsons) + name = prefix + 'qmp_schema_json' + self.decl = mcgen(''' +extern const char %(c_name)s[]; +''', + c_name=c_name(name)) + lines = to_json(jsons).split('\n') + c_string = '\n '.join([to_c_string(line) for line in lines]) + self.defn = mcgen(''' +const char %(c_name)s[] = %(c_string)s; +''', + c_name=c_name(name), + c_string=c_string) + self._schema = None + self._jsons = None + self._used_types = None + self._name_map = None + + def visit_needed(self, entity): + # Ignore types on first pass; visit_end() will pick up used types + return not isinstance(entity, QAPISchemaType) + + def _name(self, name): + if self._unmask: + return name + if name not in self._name_map: + self._name_map[name] = '%d' % len(self._name_map) + return self._name_map[name] + + def _use_type(self, typ): + # Map the various integer types to plain int + if typ.json_type() == 'int': + typ = self._schema.lookup_type('int') + elif (isinstance(typ, QAPISchemaArrayType) and + typ.element_type.json_type() == 'int'): + typ = self._schema.lookup_type('intList') + # Add type to work queue if new + if typ not in self._used_types: + self._used_types.append(typ) + # Clients should examine commands and events, not types. Hide + # type names to reduce the temptation. Also saves a few + # characters. + if isinstance(typ, QAPISchemaBuiltinType): + return typ.name + if isinstance(typ, QAPISchemaArrayType): + return '[' + self._use_type(typ.element_type) + ']' + return self._name(typ.name) + + def _gen_json(self, name, mtype, obj): + if mtype not in ('command', 'event', 'builtin', 'array'): + name = self._name(name) + obj['name'] = name + obj['meta-type'] = mtype + self._jsons.append(obj) + + def _gen_member(self, member): + ret = {'name': member.name, 'type': self._use_type(member.type)} + if member.optional: + ret['default'] = None + return ret + + def _gen_variants(self, tag_name, variants): + return {'tag': tag_name, + 'variants': [self._gen_variant(v) for v in variants]} + + def _gen_variant(self, variant): + return {'case': variant.name, 'type': self._use_type(variant.type)} + + def visit_builtin_type(self, name, info, json_type): + self._gen_json(name, 'builtin', {'json-type': json_type}) + + def visit_enum_type(self, name, info, values, prefix): + self._gen_json(name, 'enum', {'values': values}) + + def visit_array_type(self, name, info, element_type): + element = self._use_type(element_type) + self._gen_json('[' + element + ']', 'array', {'element-type': element}) + + def visit_object_type_flat(self, name, info, members, variants): + obj = {'members': [self._gen_member(m) for m in members]} + if variants: + obj.update(self._gen_variants(variants.tag_member.name, + variants.variants)) + self._gen_json(name, 'object', obj) + + def visit_alternate_type(self, name, info, variants): + self._gen_json(name, 'alternate', + {'members': [{'type': self._use_type(m.type)} + for m in variants.variants]}) + + def visit_command(self, name, info, arg_type, ret_type, + gen, success_response): + arg_type = arg_type or self._schema.the_empty_object_type + ret_type = ret_type or self._schema.the_empty_object_type + self._gen_json(name, 'command', + {'arg-type': self._use_type(arg_type), + 'ret-type': self._use_type(ret_type)}) + + def visit_event(self, name, info, arg_type): + arg_type = arg_type or self._schema.the_empty_object_type + self._gen_json(name, 'event', {'arg-type': self._use_type(arg_type)}) + +# Debugging aid: unmask QAPI schema's type names +# We normally mask them, because they're not QMP wire ABI +opt_unmask = False + +(input_file, output_dir, do_c, do_h, prefix, opts) = \ + parse_command_line("u", ["unmask-non-abi-names"]) + +for o, a in opts: + if o in ("-u", "--unmask-non-abi-names"): + opt_unmask = True + +c_comment = ''' +/* + * QAPI/QMP schema introspection + * + * Copyright (C) 2015 Red Hat, Inc. + * + * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. + * See the COPYING.LIB file in the top-level directory. + * + */ +''' +h_comment = ''' +/* + * QAPI/QMP schema introspection + * + * Copyright (C) 2015 Red Hat, Inc. + * + * This work is licensed under the terms of the GNU LGPL, version 2.1 or later. + * See the COPYING.LIB file in the top-level directory. + * + */ +''' + +(fdef, fdecl) = open_output(output_dir, do_c, do_h, prefix, + 'qmp-introspect.c', 'qmp-introspect.h', + c_comment, h_comment) + +fdef.write(mcgen(''' +#include "qemu/osdep.h" +#include "%(prefix)sqmp-introspect.h" + +''', + prefix=prefix)) + +schema = QAPISchema(input_file) +gen = QAPISchemaGenIntrospectVisitor(opt_unmask) +schema.visit(gen) +fdef.write(gen.defn) +fdecl.write(gen.decl) + +close_output(fdef, fdecl) diff --git a/qemu/scripts/qapi-types.py b/qemu/scripts/qapi-types.py index e6eb4b613..437cf6c8e 100644 --- a/qemu/scripts/qapi-types.py +++ b/qemu/scripts/qapi-types.py @@ -2,271 +2,247 @@ # QAPI types generator # # Copyright IBM, Corp. 2011 +# Copyright (c) 2013-2016 Red Hat Inc. # # Authors: # Anthony Liguori <aliguori@us.ibm.com> +# Markus Armbruster <armbru@redhat.com> # # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. -from ordereddict import OrderedDict from qapi import * -def generate_fwd_builtin(name): - return mcgen(''' -typedef struct %(name)sList -{ - union { - %(type)s value; - uint64_t padding; - }; - struct %(name)sList *next; -} %(name)sList; -''', - type=c_type(name), - name=name) +# variants must be emitted before their container; track what has already +# been output +objects_seen = set() -def generate_fwd_struct(name): - return mcgen(''' -typedef struct %(name)s %(name)s; +def gen_fwd_object_or_array(name): + return mcgen(''' -typedef struct %(name)sList -{ - union { - %(name)s *value; - uint64_t padding; - }; - struct %(name)sList *next; -} %(name)sList; +typedef struct %(c_name)s %(c_name)s; ''', - name=c_name(name)) + c_name=c_name(name)) + -def generate_fwd_enum_struct(name): +def gen_array(name, element_type): return mcgen(''' -typedef struct %(name)sList -{ - union { - %(name)s value; - uint64_t padding; - }; - struct %(name)sList *next; -} %(name)sList; + +struct %(c_name)s { + %(c_name)s *next; + %(c_type)s value; +}; ''', - name=c_name(name)) + c_name=c_name(name), c_type=element_type.c_type()) -def generate_struct_fields(members): - ret = '' - for argname, argentry, optional in parse_args(members): - if optional: +def gen_struct_members(members): + ret = '' + for memb in members: + if memb.optional: ret += mcgen(''' bool has_%(c_name)s; ''', - c_name=c_name(argname)) + c_name=c_name(memb.name)) ret += mcgen(''' %(c_type)s %(c_name)s; ''', - c_type=c_type(argentry), c_name=c_name(argname)) - + c_type=memb.type.c_type(), c_name=c_name(memb.name)) return ret -def generate_struct(expr): - structname = expr.get('struct', "") - members = expr['data'] - base = expr.get('base') +def gen_object(name, base, members, variants): + if name in objects_seen: + return '' + objects_seen.add(name) - ret = mcgen(''' -struct %(name)s -{ -''', - name=c_name(structname)) + ret = '' + if variants: + for v in variants.variants: + if isinstance(v.type, QAPISchemaObjectType): + ret += gen_object(v.type.name, v.type.base, + v.type.local_members, v.type.variants) - if base: - ret += generate_struct_fields({'base': base}) + ret += mcgen(''' - ret += generate_struct_fields(members) +struct %(c_name)s { +''', + c_name=c_name(name)) - # Make sure that all structs have at least one field; this avoids - # potential issues with attempting to malloc space for zero-length structs - # in C, and also incompatibility with C++ (where an empty struct is size 1). - if not base and not members: + if base: + if not base.is_implicit(): ret += mcgen(''' - char qapi_dummy_field_for_empty_struct; -''') - - ret += mcgen(''' -}; + /* Members inherited from %(c_name)s: */ +''', + c_name=base.c_name()) + ret += gen_struct_members(base.members) + if not base.is_implicit(): + ret += mcgen(''' + /* Own members: */ ''') + ret += gen_struct_members(members) - return ret + if variants: + ret += gen_variants(variants) -def generate_enum_lookup(name, values): - ret = mcgen(''' -const char * const %(name)s_lookup[] = { -''', - name=c_name(name)) - i = 0 - for value in values: - index = c_enum_const(name, value) + # Make sure that all structs have at least one member; this avoids + # potential issues with attempting to malloc space for zero-length + # structs in C, and also incompatibility with C++ (where an empty + # struct is size 1). + if not (base and base.members) and not members and not variants: ret += mcgen(''' - [%(index)s] = "%(value)s", -''', - index = index, value = value) + char qapi_dummy_for_empty_struct; +''') - max_index = c_enum_const(name, 'MAX') ret += mcgen(''' - [%(max_index)s] = NULL, }; +''') -''', - max_index=max_index) return ret -def generate_enum(name, values): - name = c_name(name) - lookup_decl = mcgen(''' -extern const char * const %(name)s_lookup[]; -''', - name=name) - - enum_decl = mcgen(''' -typedef enum %(name)s -{ -''', - name=name) - # append automatically generated _MAX value - enum_values = values + [ 'MAX' ] - - i = 0 - for value in enum_values: - enum_full_value = c_enum_const(name, value) - enum_decl += mcgen(''' - %(enum_full_value)s = %(i)d, -''', - enum_full_value = enum_full_value, - i=i) - i += 1 +def gen_upcast(name, base): + # C makes const-correctness ugly. We have to cast away const to let + # this function work for both const and non-const obj. + return mcgen(''' - enum_decl += mcgen(''' -} %(name)s; +static inline %(base)s *qapi_%(c_name)s_base(const %(c_name)s *obj) +{ + return (%(base)s *)obj; +} ''', - name=name) + c_name=c_name(name), base=base.c_name()) - return lookup_decl + enum_decl - -def generate_alternate_qtypes(expr): - - name = expr['alternate'] - members = expr['data'] +def gen_variants(variants): ret = mcgen(''' -const int %(name)s_qtypes[QTYPE_MAX] = { + union { /* union tag is @%(c_name)s */ ''', - name=c_name(name)) - - for key in members: - qtype = find_alternate_member_qtype(members[key]) - assert qtype, "Invalid alternate member" + c_name=c_name(variants.tag_member.name)) + for var in variants.variants: ret += mcgen(''' - [%(qtype)s] = %(enum_const)s, + %(c_type)s %(c_name)s; ''', - qtype = qtype, - enum_const = c_enum_const(name + 'Kind', key)) + c_type=var.type.c_unboxed_type(), + c_name=c_name(var.name)) ret += mcgen(''' -}; + } u; ''') - return ret - - -def generate_union(expr, meta): - name = c_name(expr[meta]) - typeinfo = expr['data'] - - base = expr.get('base') - discriminator = expr.get('discriminator') + return ret - enum_define = discriminator_find_enum_define(expr) - if enum_define: - discriminator_type_name = enum_define['enum_name'] - else: - discriminator_type_name = '%sKind' % (name) +def gen_type_cleanup_decl(name): ret = mcgen(''' -struct %(name)s -{ - %(discriminator_type_name)s kind; - union { - void *data; -''', - name=name, - discriminator_type_name=c_name(discriminator_type_name)) - for key in typeinfo: - ret += mcgen(''' - %(c_type)s %(c_name)s; +void qapi_free_%(c_name)s(%(c_name)s *obj); ''', - c_type=c_type(typeinfo[key]), - c_name=c_name(key)) - - ret += mcgen(''' - }; -''') - - if base: - assert discriminator - base_fields = find_struct(base)['data'].copy() - del base_fields[discriminator] - ret += generate_struct_fields(base_fields) - else: - assert not discriminator - - ret += mcgen(''' -}; -''') - if meta == 'alternate': - ret += mcgen(''' -extern const int %(name)s_qtypes[]; -''', - name=name) - - + c_name=c_name(name)) return ret -def generate_type_cleanup_decl(name): - ret = mcgen(''' -void qapi_free_%(name)s(%(c_type)s obj); -''', - c_type=c_type(name), name=c_name(name)) - return ret -def generate_type_cleanup(name): +def gen_type_cleanup(name): ret = mcgen(''' -void qapi_free_%(name)s(%(c_type)s obj) +void qapi_free_%(c_name)s(%(c_name)s *obj) { - QapiDeallocVisitor *md; + QapiDeallocVisitor *qdv; Visitor *v; if (!obj) { return; } - md = qapi_dealloc_visitor_new(); - v = qapi_dealloc_get_visitor(md); - visit_type_%(name)s(v, &obj, NULL, NULL); - qapi_dealloc_visitor_cleanup(md); + qdv = qapi_dealloc_visitor_new(); + v = qapi_dealloc_get_visitor(qdv); + visit_type_%(c_name)s(v, NULL, &obj, NULL); + qapi_dealloc_visitor_cleanup(qdv); } ''', - c_type=c_type(name), name=c_name(name)) + c_name=c_name(name)) return ret + +class QAPISchemaGenTypeVisitor(QAPISchemaVisitor): + def __init__(self): + self.decl = None + self.defn = None + self._fwdecl = None + self._btin = None + + def visit_begin(self, schema): + # gen_object() is recursive, ensure it doesn't visit the empty type + objects_seen.add(schema.the_empty_object_type.name) + self.decl = '' + self.defn = '' + self._fwdecl = '' + self._btin = guardstart('QAPI_TYPES_BUILTIN') + + def visit_end(self): + self.decl = self._fwdecl + self.decl + self._fwdecl = None + # To avoid header dependency hell, we always generate + # declarations for built-in types in our header files and + # simply guard them. See also do_builtins (command line + # option -b). + self._btin += guardend('QAPI_TYPES_BUILTIN') + self.decl = self._btin + self.decl + self._btin = None + + def _gen_type_cleanup(self, name): + self.decl += gen_type_cleanup_decl(name) + self.defn += gen_type_cleanup(name) + + def visit_enum_type(self, name, info, values, prefix): + # Special case for our lone builtin enum type + # TODO use something cleaner than existence of info + if not info: + self._btin += gen_enum(name, values, prefix) + if do_builtins: + self.defn += gen_enum_lookup(name, values, prefix) + else: + self._fwdecl += gen_enum(name, values, prefix) + self.defn += gen_enum_lookup(name, values, prefix) + + def visit_array_type(self, name, info, element_type): + if isinstance(element_type, QAPISchemaBuiltinType): + self._btin += gen_fwd_object_or_array(name) + self._btin += gen_array(name, element_type) + self._btin += gen_type_cleanup_decl(name) + if do_builtins: + self.defn += gen_type_cleanup(name) + else: + self._fwdecl += gen_fwd_object_or_array(name) + self.decl += gen_array(name, element_type) + self._gen_type_cleanup(name) + + def visit_object_type(self, name, info, base, members, variants): + # Nothing to do for the special empty builtin + if name == 'q_empty': + return + self._fwdecl += gen_fwd_object_or_array(name) + self.decl += gen_object(name, base, members, variants) + if base and not base.is_implicit(): + self.decl += gen_upcast(name, base) + # TODO Worth changing the visitor signature, so we could + # directly use rather than repeat type.is_implicit()? + if not name.startswith('q_'): + # implicit types won't be directly allocated/freed + self._gen_type_cleanup(name) + + def visit_alternate_type(self, name, info, variants): + self._fwdecl += gen_fwd_object_or_array(name) + self.decl += gen_object(name, None, [variants.tag_member], variants) + self._gen_type_cleanup(name) + +# If you link code generated from multiple schemata, you want only one +# instance of the code for built-in types. Generate it only when +# do_builtins, enabled by command line option -b. See also +# QAPISchemaGenTypeVisitor.visit_end(). do_builtins = False (input_file, output_dir, do_c, do_h, prefix, opts) = \ @@ -311,92 +287,17 @@ h_comment = ''' c_comment, h_comment) fdef.write(mcgen(''' +#include "qemu/osdep.h" #include "qapi/dealloc-visitor.h" #include "%(prefix)sqapi-types.h" #include "%(prefix)sqapi-visit.h" - ''', prefix=prefix)) -fdecl.write(mcgen(''' -#include <stdbool.h> -#include <stdint.h> - -''')) - -exprs = parse_schema(input_file) - -fdecl.write(guardstart("QAPI_TYPES_BUILTIN_STRUCT_DECL")) -for typename in builtin_types.keys(): - fdecl.write(generate_fwd_builtin(typename)) -fdecl.write(guardend("QAPI_TYPES_BUILTIN_STRUCT_DECL")) - -for expr in exprs: - ret = "\n" - if expr.has_key('struct'): - ret += generate_fwd_struct(expr['struct']) - elif expr.has_key('enum'): - ret += generate_enum(expr['enum'], expr['data']) + "\n" - ret += generate_fwd_enum_struct(expr['enum']) - fdef.write(generate_enum_lookup(expr['enum'], expr['data'])) - elif expr.has_key('union'): - ret += generate_fwd_struct(expr['union']) + "\n" - enum_define = discriminator_find_enum_define(expr) - if not enum_define: - ret += generate_enum('%sKind' % expr['union'], expr['data'].keys()) - fdef.write(generate_enum_lookup('%sKind' % expr['union'], - expr['data'].keys())) - elif expr.has_key('alternate'): - ret += generate_fwd_struct(expr['alternate']) + "\n" - ret += generate_enum('%sKind' % expr['alternate'], expr['data'].keys()) - fdef.write(generate_enum_lookup('%sKind' % expr['alternate'], - expr['data'].keys())) - fdef.write(generate_alternate_qtypes(expr)) - else: - continue - fdecl.write(ret) - -# to avoid header dependency hell, we always generate declarations -# for built-in types in our header files and simply guard them -fdecl.write(guardstart("QAPI_TYPES_BUILTIN_CLEANUP_DECL")) -for typename in builtin_types.keys(): - fdecl.write(generate_type_cleanup_decl(typename + "List")) -fdecl.write(guardend("QAPI_TYPES_BUILTIN_CLEANUP_DECL")) - -# ...this doesn't work for cases where we link in multiple objects that -# have the functions defined, so we use -b option to provide control -# over these cases -if do_builtins: - fdef.write(guardstart("QAPI_TYPES_BUILTIN_CLEANUP_DEF")) - for typename in builtin_types.keys(): - fdef.write(generate_type_cleanup(typename + "List")) - fdef.write(guardend("QAPI_TYPES_BUILTIN_CLEANUP_DEF")) - -for expr in exprs: - ret = "\n" - if expr.has_key('struct'): - ret += generate_struct(expr) + "\n" - ret += generate_type_cleanup_decl(expr['struct'] + "List") - fdef.write(generate_type_cleanup(expr['struct'] + "List") + "\n") - ret += generate_type_cleanup_decl(expr['struct']) - fdef.write(generate_type_cleanup(expr['struct']) + "\n") - elif expr.has_key('union'): - ret += generate_union(expr, 'union') - ret += generate_type_cleanup_decl(expr['union'] + "List") - fdef.write(generate_type_cleanup(expr['union'] + "List") + "\n") - ret += generate_type_cleanup_decl(expr['union']) - fdef.write(generate_type_cleanup(expr['union']) + "\n") - elif expr.has_key('alternate'): - ret += generate_union(expr, 'alternate') - ret += generate_type_cleanup_decl(expr['alternate'] + "List") - fdef.write(generate_type_cleanup(expr['alternate'] + "List") + "\n") - ret += generate_type_cleanup_decl(expr['alternate']) - fdef.write(generate_type_cleanup(expr['alternate']) + "\n") - elif expr.has_key('enum'): - ret += generate_type_cleanup_decl(expr['enum'] + "List") - fdef.write(generate_type_cleanup(expr['enum'] + "List") + "\n") - else: - continue - fdecl.write(ret) +schema = QAPISchema(input_file) +gen = QAPISchemaGenTypeVisitor() +schema.visit(gen) +fdef.write(gen.defn) +fdecl.write(gen.decl) close_output(fdef, fdecl) diff --git a/qemu/scripts/qapi-visit.py b/qemu/scripts/qapi-visit.py index 5b9933648..31d233035 100644 --- a/qemu/scripts/qapi-visit.py +++ b/qemu/scripts/qapi-visit.py @@ -2,7 +2,7 @@ # QAPI visitor generator # # Copyright IBM, Corp. 2011 -# Copyright (C) 2014-2015 Red Hat, Inc. +# Copyright (C) 2014-2016 Red Hat, Inc. # # Authors: # Anthony Liguori <aliguori@us.ibm.com> @@ -12,86 +12,90 @@ # This work is licensed under the terms of the GNU GPL, version 2. # See the COPYING file in the top-level directory. -from ordereddict import OrderedDict from qapi import * import re -implicit_structs = [] -def generate_visit_implicit_struct(type): - global implicit_structs - if type in implicit_structs: - return '' - implicit_structs.append(type) +def gen_visit_decl(name, scalar=False): + c_type = c_name(name) + ' *' + if not scalar: + c_type += '*' return mcgen(''' +void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_type)sobj, Error **errp); +''', + c_name=c_name(name), c_type=c_type) -static void visit_type_implicit_%(c_type)s(Visitor *m, %(c_type)s **obj, Error **errp) -{ - Error *err = NULL; - visit_start_implicit_struct(m, (void **)obj, sizeof(%(c_type)s), &err); - if (!err) { - visit_type_%(c_type)s_fields(m, obj, errp); - visit_end_implicit_struct(m, &err); - } - error_propagate(errp, err); -} +def gen_visit_members_decl(name): + return mcgen(''' + +void visit_type_%(c_name)s_members(Visitor *v, %(c_name)s *obj, Error **errp); ''', - c_type=type_name(type)) + c_name=c_name(name)) -def generate_visit_struct_fields(name, members, base = None): - substructs = [] - ret = '' - if base: - ret += generate_visit_implicit_struct(base) - - ret += mcgen(''' +def gen_visit_object_members(name, base, members, variants): + ret = mcgen(''' -static void visit_type_%(name)s_fields(Visitor *m, %(name)s **obj, Error **errp) +void visit_type_%(c_name)s_members(Visitor *v, %(c_name)s *obj, Error **errp) { Error *err = NULL; + ''', - name=c_name(name)) - push_indent() + c_name=c_name(name)) if base: ret += mcgen(''' -visit_type_implicit_%(type)s(m, &(*obj)->%(c_name)s, &err); -if (err) { - goto out; -} + visit_type_%(c_type)s_members(v, (%(c_type)s *)obj, &err); ''', - type=type_name(base), c_name=c_name('base')) + c_type=base.c_name()) + ret += gen_err_check() - for argname, argentry, optional in parse_args(members): - if optional: + for memb in members: + if memb.optional: ret += mcgen(''' -visit_optional(m, &(*obj)->has_%(c_name)s, "%(name)s", &err); -if (!err && (*obj)->has_%(c_name)s) { + if (visit_optional(v, "%(name)s", &obj->has_%(c_name)s)) { ''', - c_name=c_name(argname), name=argname) + name=memb.name, c_name=c_name(memb.name)) push_indent() - ret += mcgen(''' -visit_type_%(type)s(m, &(*obj)->%(c_name)s, "%(name)s", &err); + visit_type_%(c_type)s(v, "%(name)s", &obj->%(c_name)s, &err); ''', - type=type_name(argentry), c_name=c_name(argname), - name=argname) - - if optional: + c_type=memb.type.c_name(), name=memb.name, + c_name=c_name(memb.name)) + ret += gen_err_check() + if memb.optional: pop_indent() ret += mcgen(''' -} + } ''') + + if variants: ret += mcgen(''' -if (err) { - goto out; -} + switch (obj->%(c_name)s) { +''', + c_name=c_name(variants.tag_member.name)) + + for var in variants.variants: + ret += mcgen(''' + case %(case)s: + visit_type_%(c_type)s_members(v, &obj->u.%(c_name)s, &err); + break; +''', + case=c_enum_const(variants.tag_member.type.name, + var.name, + variants.tag_member.type.prefix), + c_type=var.type.c_name(), c_name=c_name(var.name)) + + ret += mcgen(''' + default: + abort(); + } ''') - pop_indent() - if re.search('^ *goto out\\;', ret, re.MULTILINE): + # 'goto out' produced for base, for each member, and if variants were + # present + if base or members or variants: ret += mcgen(''' out: @@ -103,275 +107,209 @@ out: return ret -def generate_visit_struct_body(name, members): - ret = mcgen(''' - Error *err = NULL; - - visit_start_struct(m, (void **)obj, "%(name)s", name, sizeof(%(c_name)s), &err); - if (!err) { - if (*obj) { - visit_type_%(c_name)s_fields(m, obj, errp); - } - visit_end_struct(m, &err); - } - error_propagate(errp, err); -''', - name=name, c_name=c_name(name)) - - return ret - -def generate_visit_struct(expr): - - name = expr['struct'] - members = expr['data'] - base = expr.get('base') - - ret = generate_visit_struct_fields(name, members, base) - - ret += mcgen(''' - -void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp) -{ -''', - name=c_name(name)) - - ret += generate_visit_struct_body(name, members) - - ret += mcgen(''' -} -''') - return ret - -def generate_visit_list(name, members): +def gen_visit_list(name, element_type): + # FIXME: if *obj is NULL on entry, and the first visit_next_list() + # assigns to *obj, while a later one fails, we should clean up *obj + # rather than leaving it non-NULL. As currently written, the caller must + # call qapi_free_FOOList() to avoid a memory leak of the partial FOOList. return mcgen(''' -void visit_type_%(name)sList(Visitor *m, %(name)sList **obj, const char *name, Error **errp) +void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp) { Error *err = NULL; GenericList *i, **prev; - visit_start_list(m, name, &err); + visit_start_list(v, name, &err); if (err) { goto out; } for (prev = (GenericList **)obj; - !err && (i = visit_next_list(m, prev, &err)) != NULL; + !err && (i = visit_next_list(v, prev, sizeof(**obj))) != NULL; prev = &i) { - %(name)sList *native_i = (%(name)sList *)i; - visit_type_%(name)s(m, &native_i->value, NULL, &err); + %(c_name)s *native_i = (%(c_name)s *)i; + visit_type_%(c_elt_type)s(v, NULL, &native_i->value, &err); } - error_propagate(errp, err); - err = NULL; - visit_end_list(m, &err); + visit_end_list(v); out: error_propagate(errp, err); } ''', - name=type_name(name)) + c_name=c_name(name), c_elt_type=element_type.c_name()) -def generate_visit_enum(name, members): + +def gen_visit_enum(name): return mcgen(''' -void visit_type_%(name)s(Visitor *m, %(name)s *obj, const char *name, Error **errp) +void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s *obj, Error **errp) { - visit_type_enum(m, (int *)obj, %(name)s_lookup, "%(name)s", name, errp); + int value = *obj; + visit_type_enum(v, name, &value, %(c_name)s_lookup, errp); + *obj = value; } ''', - name=c_name(name)) - -def generate_visit_alternate(name, members): - ret = mcgen(''' - -void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp) -{ - Error *err = NULL; - - visit_start_implicit_struct(m, (void**) obj, sizeof(%(name)s), &err); - if (err) { - goto out; - } - visit_get_next_type(m, (int*) &(*obj)->kind, %(name)s_qtypes, name, &err); - if (err) { - goto out_end; - } - switch ((*obj)->kind) { -''', - name=c_name(name)) - - # For alternate, always use the default enum type automatically generated - # as name + 'Kind' - disc_type = c_name(name) + 'Kind' - - for key in members: - assert (members[key] in builtin_types.keys() - or find_struct(members[key]) - or find_union(members[key]) - or find_enum(members[key])), "Invalid alternate member" - - enum_full_value = c_enum_const(disc_type, key) - ret += mcgen(''' - case %(enum_full_value)s: - visit_type_%(c_type)s(m, &(*obj)->%(c_name)s, name, &err); - break; -''', - enum_full_value = enum_full_value, - c_type = type_name(members[key]), - c_name = c_name(key)) - - ret += mcgen(''' - default: - abort(); - } -out_end: - error_propagate(errp, err); - err = NULL; - visit_end_implicit_struct(m, &err); -out: - error_propagate(errp, err); -} -''') - - return ret - - -def generate_visit_union(expr): - - name = expr['union'] - members = expr['data'] - - base = expr.get('base') - discriminator = expr.get('discriminator') - - enum_define = discriminator_find_enum_define(expr) - if enum_define: - # Use the enum type as discriminator - ret = "" - disc_type = c_name(enum_define['enum_name']) - else: - # There will always be a discriminator in the C switch code, by default - # it is an enum type generated silently - ret = generate_visit_enum(name + 'Kind', members.keys()) - disc_type = c_name(name) + 'Kind' + c_name=c_name(name)) - if base: - assert discriminator - base_fields = find_struct(base)['data'].copy() - del base_fields[discriminator] - ret += generate_visit_struct_fields(name, base_fields) - if discriminator: - for key in members: - ret += generate_visit_implicit_struct(members[key]) +def gen_visit_alternate(name, variants): + promote_int = 'true' + ret = '' + for var in variants.variants: + if var.type.alternate_qtype() == 'QTYPE_QINT': + promote_int = 'false' ret += mcgen(''' -void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp) +void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp) { Error *err = NULL; - visit_start_struct(m, (void **)obj, "%(name)s", name, sizeof(%(name)s), &err); + visit_start_alternate(v, name, (GenericAlternate **)obj, sizeof(**obj), + %(promote_int)s, &err); if (err) { goto out; } - if (*obj) { + switch ((*obj)->type) { ''', - name=c_name(name)) + c_name=c_name(name), promote_int=promote_int) - if base: + for var in variants.variants: ret += mcgen(''' - visit_type_%(name)s_fields(m, obj, &err); - if (err) { - goto out_obj; - } + case %(case)s: ''', - name=c_name(name)) - - if not discriminator: - disc_key = "type" - else: - disc_key = discriminator - ret += mcgen(''' - visit_type_%(disc_type)s(m, &(*obj)->kind, "%(disc_key)s", &err); + case=var.type.alternate_qtype()) + if isinstance(var.type, QAPISchemaObjectType): + ret += mcgen(''' + visit_start_struct(v, name, NULL, 0, &err); if (err) { - goto out_obj; - } - if (!visit_start_union(m, !!(*obj)->data, &err) || err) { - goto out_obj; + break; } - switch ((*obj)->kind) { + visit_type_%(c_type)s_members(v, &(*obj)->u.%(c_name)s, &err); + error_propagate(errp, err); + err = NULL; + visit_end_struct(v, &err); ''', - disc_type = disc_type, - disc_key = disc_key) - - for key in members: - if not discriminator: - fmt = 'visit_type_%(c_type)s(m, &(*obj)->%(c_name)s, "data", &err);' + c_type=var.type.c_name(), + c_name=c_name(var.name)) else: - fmt = 'visit_type_implicit_%(c_type)s(m, &(*obj)->%(c_name)s, &err);' - - enum_full_value = c_enum_const(disc_type, key) - ret += mcgen(''' - case %(enum_full_value)s: - ''' + fmt + ''' - break; + ret += mcgen(''' + visit_type_%(c_type)s(v, name, &(*obj)->u.%(c_name)s, &err); ''', - enum_full_value = enum_full_value, - c_type=type_name(members[key]), - c_name=c_name(key)) + c_type=var.type.c_name(), + c_name=c_name(var.name)) + ret += mcgen(''' + break; +''') ret += mcgen(''' - default: - abort(); - } -out_obj: - error_propagate(errp, err); - err = NULL; - visit_end_union(m, !!(*obj)->data, &err); - error_propagate(errp, err); - err = NULL; + default: + error_setg(&err, QERR_INVALID_PARAMETER_TYPE, name ? name : "null", + "%(name)s"); } - visit_end_struct(m, &err); + visit_end_alternate(v); out: error_propagate(errp, err); } -''') - - return ret - -def generate_declaration(name, members, builtin_type=False): - ret = "" - if not builtin_type: - name = c_name(name) - ret += mcgen(''' - -void visit_type_%(name)s(Visitor *m, %(name)s **obj, const char *name, Error **errp); -''', - name=name) - - ret += mcgen(''' -void visit_type_%(name)sList(Visitor *m, %(name)sList **obj, const char *name, Error **errp); ''', name=name) return ret -def generate_enum_declaration(name, members): - ret = mcgen(''' -void visit_type_%(name)sList(Visitor *m, %(name)sList **obj, const char *name, Error **errp); -''', - name=c_name(name)) - return ret - -def generate_decl_enum(name, members): +def gen_visit_object(name, base, members, variants): + # FIXME: if *obj is NULL on entry, and visit_start_struct() assigns to + # *obj, but then visit_type_FOO_members() fails, we should clean up *obj + # rather than leaving it non-NULL. As currently written, the caller must + # call qapi_free_FOO() to avoid a memory leak of the partial FOO. return mcgen(''' -void visit_type_%(name)s(Visitor *m, %(name)s *obj, const char *name, Error **errp); -''', - name=c_name(name)) +void visit_type_%(c_name)s(Visitor *v, const char *name, %(c_name)s **obj, Error **errp) +{ + Error *err = NULL; + visit_start_struct(v, name, (void **)obj, sizeof(%(c_name)s), &err); + if (err) { + goto out; + } + if (!*obj) { + goto out_obj; + } + visit_type_%(c_name)s_members(v, *obj, &err); + error_propagate(errp, err); + err = NULL; +out_obj: + visit_end_struct(v, &err); +out: + error_propagate(errp, err); +} +''', + c_name=c_name(name)) + + +class QAPISchemaGenVisitVisitor(QAPISchemaVisitor): + def __init__(self): + self.decl = None + self.defn = None + self._btin = None + + def visit_begin(self, schema): + self.decl = '' + self.defn = '' + self._btin = guardstart('QAPI_VISIT_BUILTIN') + + def visit_end(self): + # To avoid header dependency hell, we always generate + # declarations for built-in types in our header files and + # simply guard them. See also do_builtins (command line + # option -b). + self._btin += guardend('QAPI_VISIT_BUILTIN') + self.decl = self._btin + self.decl + self._btin = None + + def visit_enum_type(self, name, info, values, prefix): + # Special case for our lone builtin enum type + # TODO use something cleaner than existence of info + if not info: + self._btin += gen_visit_decl(name, scalar=True) + if do_builtins: + self.defn += gen_visit_enum(name) + else: + self.decl += gen_visit_decl(name, scalar=True) + self.defn += gen_visit_enum(name) + + def visit_array_type(self, name, info, element_type): + decl = gen_visit_decl(name) + defn = gen_visit_list(name, element_type) + if isinstance(element_type, QAPISchemaBuiltinType): + self._btin += decl + if do_builtins: + self.defn += defn + else: + self.decl += decl + self.defn += defn + + def visit_object_type(self, name, info, base, members, variants): + # Nothing to do for the special empty builtin + if name == 'q_empty': + return + self.decl += gen_visit_members_decl(name) + self.defn += gen_visit_object_members(name, base, members, variants) + # TODO Worth changing the visitor signature, so we could + # directly use rather than repeat type.is_implicit()? + if not name.startswith('q_'): + # only explicit types need an allocating visit + self.decl += gen_visit_decl(name) + self.defn += gen_visit_object(name, base, members, variants) + + def visit_alternate_type(self, name, info, variants): + self.decl += gen_visit_decl(name) + self.defn += gen_visit_alternate(name, variants) + +# If you link code generated from multiple schemata, you want only one +# instance of the code for built-in types. Generate it only when +# do_builtins, enabled by command line option -b. See also +# QAPISchemaGenVisitVisitor.visit_end(). do_builtins = False (input_file, output_dir, do_c, do_h, prefix, opts) = \ @@ -415,70 +353,25 @@ h_comment = ''' c_comment, h_comment) fdef.write(mcgen(''' +#include "qemu/osdep.h" #include "qemu-common.h" +#include "qapi/error.h" #include "%(prefix)sqapi-visit.h" ''', - prefix = prefix)) + prefix=prefix)) fdecl.write(mcgen(''' #include "qapi/visitor.h" +#include "qapi/qmp/qerror.h" #include "%(prefix)sqapi-types.h" ''', prefix=prefix)) -exprs = parse_schema(input_file) - -# to avoid header dependency hell, we always generate declarations -# for built-in types in our header files and simply guard them -fdecl.write(guardstart("QAPI_VISIT_BUILTIN_VISITOR_DECL")) -for typename in builtin_types.keys(): - fdecl.write(generate_declaration(typename, None, builtin_type=True)) -fdecl.write(guardend("QAPI_VISIT_BUILTIN_VISITOR_DECL")) - -# ...this doesn't work for cases where we link in multiple objects that -# have the functions defined, so we use -b option to provide control -# over these cases -if do_builtins: - for typename in builtin_types.keys(): - fdef.write(generate_visit_list(typename, None)) - -for expr in exprs: - if expr.has_key('struct'): - ret = generate_visit_struct(expr) - ret += generate_visit_list(expr['struct'], expr['data']) - fdef.write(ret) - - ret = generate_declaration(expr['struct'], expr['data']) - fdecl.write(ret) - elif expr.has_key('union'): - ret = generate_visit_union(expr) - ret += generate_visit_list(expr['union'], expr['data']) - fdef.write(ret) - - enum_define = discriminator_find_enum_define(expr) - ret = "" - if not enum_define: - ret = generate_decl_enum('%sKind' % expr['union'], - expr['data'].keys()) - ret += generate_declaration(expr['union'], expr['data']) - fdecl.write(ret) - elif expr.has_key('alternate'): - ret = generate_visit_alternate(expr['alternate'], expr['data']) - ret += generate_visit_list(expr['alternate'], expr['data']) - fdef.write(ret) - - ret = generate_decl_enum('%sKind' % expr['alternate'], - expr['data'].keys()) - ret += generate_declaration(expr['alternate'], expr['data']) - fdecl.write(ret) - elif expr.has_key('enum'): - ret = generate_visit_list(expr['enum'], expr['data']) - ret += generate_visit_enum(expr['enum'], expr['data']) - fdef.write(ret) - - ret = generate_decl_enum(expr['enum'], expr['data']) - ret += generate_enum_declaration(expr['enum'], expr['data']) - fdecl.write(ret) +schema = QAPISchema(input_file) +gen = QAPISchemaGenVisitVisitor() +schema.visit(gen) +fdef.write(gen.defn) +fdecl.write(gen.decl) close_output(fdef, fdecl) diff --git a/qemu/scripts/qapi.py b/qemu/scripts/qapi.py index 06d7fc284..b13ae4789 100644 --- a/qemu/scripts/qapi.py +++ b/qemu/scripts/qapi.py @@ -2,7 +2,7 @@ # QAPI helper library # # Copyright IBM, Corp. 2011 -# Copyright (c) 2013-2015 Red Hat Inc. +# Copyright (c) 2013-2016 Red Hat Inc. # # Authors: # Anthony Liguori <aliguori@us.ibm.com> @@ -33,12 +33,15 @@ builtin_types = { 'uint32': 'QTYPE_QINT', 'uint64': 'QTYPE_QINT', 'size': 'QTYPE_QINT', + 'any': None, # any QType possible, actually + 'QType': 'QTYPE_QSTRING', } # Whitelist of commands allowed to return a non-dictionary returns_whitelist = [ # From QMP: 'human-monitor-command', + 'qom-get', 'query-migrate-cache-size', 'query-tpm-models', 'query-tpm-types', @@ -54,9 +57,18 @@ returns_whitelist = [ 'guest-set-vcpus', 'guest-sync', 'guest-sync-delimited', +] - # From qapi-schema-test: - 'user_def_cmd3', +# Whitelist of entities allowed to violate case conventions +case_whitelist = [ + # From QMP: + 'ACPISlotType', # DIMM, visible through query-acpi-ospm-status + 'CpuInfoMIPS', # PC, visible through query-cpu + 'CpuInfoTricore', # PC, visible through query-cpu + 'QapiErrorClass', # all members, visible through errors + 'UuidInfo', # UUID, visible through query-uuid + 'X86CPURegister32', # all members, visible indirectly through qom-get + 'q_obj_CpuInfo-base', # CPU, visible through query-cpu ] enum_types = [] @@ -69,6 +81,7 @@ all_names = {} # Parsing the schema into expressions # + def error_path(parent): res = "" while parent: @@ -77,8 +90,10 @@ def error_path(parent): parent = parent['parent'] return res + class QAPISchemaError(Exception): def __init__(self, schema, msg): + Exception.__init__(self) self.fname = schema.fname self.msg = msg self.col = 1 @@ -94,8 +109,11 @@ class QAPISchemaError(Exception): return error_path(self.info) + \ "%s:%d:%d: %s" % (self.fname, self.line, self.col, self.msg) + class QAPIExprError(Exception): def __init__(self, expr_info, msg): + Exception.__init__(self) + assert expr_info self.info = expr_info self.msg = msg @@ -103,9 +121,10 @@ class QAPIExprError(Exception): return error_path(self.info['parent']) + \ "%s:%d: %s" % (self.info['file'], self.info['line'], self.msg) -class QAPISchema: - def __init__(self, fp, previously_included = [], incl_info = None): +class QAPISchemaParser(object): + + def __init__(self, fp, previously_included=[], incl_info=None): abs_fname = os.path.abspath(fp.name) fname = fp.name self.fname = fname @@ -120,18 +139,18 @@ class QAPISchema: self.exprs = [] self.accept() - while self.tok != None: + while self.tok is not None: expr_info = {'file': fname, 'line': self.line, 'parent': self.incl_info} expr = self.get_expr(False) if isinstance(expr, dict) and "include" in expr: if len(expr) != 1: - raise QAPIExprError(expr_info, "Invalid 'include' directive") + raise QAPIExprError(expr_info, + "Invalid 'include' directive") include = expr["include"] if not isinstance(include, str): raise QAPIExprError(expr_info, - 'Expected a file name (string), got: %s' - % include) + "Value of 'include' must be a string") incl_abs_fname = os.path.join(os.path.dirname(abs_fname), include) # catch inclusion cycle @@ -146,11 +165,11 @@ class QAPISchema: continue try: fobj = open(incl_abs_fname, 'r') - except IOError, e: + except IOError as e: raise QAPIExprError(expr_info, '%s: %s' % (e.strerror, include)) - exprs_include = QAPISchema(fobj, previously_included, - expr_info) + exprs_include = QAPISchemaParser(fobj, previously_included, + expr_info) self.exprs.extend(exprs_include.exprs) else: expr_elem = {'expr': expr, @@ -166,7 +185,7 @@ class QAPISchema: if self.tok == '#': self.cursor = self.src.find('\n', self.cursor) - elif self.tok in ['{', '}', ':', ',', '[', ']']: + elif self.tok in "{}:,[]": return elif self.tok == "'": string = '' @@ -190,7 +209,7 @@ class QAPISchema: string += '\t' elif ch == 'u': value = 0 - for x in range(0, 4): + for _ in range(0, 4): ch = self.src[self.cursor] self.cursor += 1 if ch not in "0123456789abcdefABCDEF": @@ -212,7 +231,7 @@ class QAPISchema: string += ch else: raise QAPISchemaError(self, - "Unknown escape \\%s" %ch) + "Unknown escape \\%s" % ch) esc = False elif ch == "\\": esc = True @@ -272,7 +291,7 @@ class QAPISchema: if self.tok == ']': self.accept() return expr - if not self.tok in "{['tfn": + if self.tok not in "{['tfn": raise QAPISchemaError(self, 'Expected "{", "[", "]", string, ' 'boolean or "null"') while True: @@ -302,17 +321,23 @@ class QAPISchema: # # Semantic analysis of schema expressions +# TODO fold into QAPISchema +# TODO catching name collisions in generated code would be nice # -def find_base_fields(base): + +def find_base_members(base): + if isinstance(base, dict): + return base base_struct_define = find_struct(base) if not base_struct_define: return None return base_struct_define['data'] + # Return the qtype of an alternate branch, or None on error. def find_alternate_member_qtype(qapi_type): - if builtin_types.has_key(qapi_type): + if qapi_type in builtin_types: return builtin_types[qapi_type] elif find_struct(qapi_type): return "QTYPE_QDICT" @@ -322,6 +347,7 @@ def find_alternate_member_qtype(qapi_type): return "QTYPE_QDICT" return None + # Return the discriminator enum define if discriminator is specified as an # enum type, otherwise return None. def discriminator_find_enum_define(expr): @@ -331,19 +357,26 @@ def discriminator_find_enum_define(expr): if not (discriminator and base): return None - base_fields = find_base_fields(base) - if not base_fields: + base_members = find_base_members(base) + if not base_members: return None - discriminator_type = base_fields.get(discriminator) + discriminator_type = base_members.get(discriminator) if not discriminator_type: return None return find_enum(discriminator_type) -valid_name = re.compile('^[a-zA-Z_][a-zA-Z0-9_.-]*$') -def check_name(expr_info, source, name, allow_optional = False, - enum_member = False): + +# Names must be letters, numbers, -, and _. They must start with letter, +# except for downstream extensions which must start with __RFQDN_. +# Dots are only valid in the downstream extension prefix. +valid_name = re.compile('^(__[a-zA-Z0-9.-]+_)?' + '[a-zA-Z][a-zA-Z0-9_-]*$') + + +def check_name(expr_info, source, name, allow_optional=False, + enum_member=False): global valid_name membername = name @@ -358,31 +391,39 @@ def check_name(expr_info, source, name, allow_optional = False, % (source, name)) # Enum members can start with a digit, because the generated C # code always prefixes it with the enum name - if enum_member: - membername = '_' + membername - if not valid_name.match(membername): + if enum_member and membername[0].isdigit(): + membername = 'D' + membername + # Reserve the entire 'q_' namespace for c_name(), and for 'q_empty' + # and 'q_obj_*' implicit type names. + if not valid_name.match(membername) or \ + c_name(membername, False).startswith('q_'): raise QAPIExprError(expr_info, "%s uses invalid name '%s'" % (source, name)) -def add_name(name, info, meta, implicit = False): + +def add_name(name, info, meta, implicit=False): global all_names check_name(info, "'%s'" % meta, name) + # FIXME should reject names that differ only in '_' vs. '.' + # vs. '-', because they're liable to clash in generated C. if name in all_names: raise QAPIExprError(info, "%s '%s' is already defined" % (all_names[name], name)) - if not implicit and name[-4:] == 'Kind': + if not implicit and (name.endswith('Kind') or name.endswith('List')): raise QAPIExprError(info, - "%s '%s' should not end in 'Kind'" - % (meta, name)) + "%s '%s' should not end in '%s'" + % (meta, name, name[-4:])) all_names[name] = meta + def add_struct(definition, info): global struct_types name = definition['struct'] add_name(name, info, 'struct') struct_types.append(definition) + def find_struct(name): global struct_types for struct in struct_types: @@ -390,12 +431,14 @@ def find_struct(name): return struct return None + def add_union(definition, info): global union_types name = definition['union'] add_name(name, info, 'union') union_types.append(definition) + def find_union(name): global union_types for union in union_types: @@ -403,11 +446,13 @@ def find_union(name): return union return None -def add_enum(name, info, enum_values = None, implicit = False): + +def add_enum(name, info, enum_values=None, implicit=False): global enum_types add_name(name, info, 'enum', implicit) enum_types.append({"enum_name": name, "enum_values": enum_values}) + def find_enum(name): global enum_types for enum in enum_types: @@ -415,21 +460,19 @@ def find_enum(name): return enum return None + def is_enum(name): - return find_enum(name) != None + return find_enum(name) is not None + -def check_type(expr_info, source, value, allow_array = False, - allow_dict = False, allow_optional = False, - allow_star = False, allow_metas = []): +def check_type(expr_info, source, value, allow_array=False, + allow_dict=False, allow_optional=False, + allow_metas=[]): global all_names - orig_value = value if value is None: return - if allow_star and value == '**': - return - # Check if array type for value is okay if isinstance(value, list): if not allow_array: @@ -440,103 +483,79 @@ def check_type(expr_info, source, value, allow_array = False, "%s: array type must contain single type name" % source) value = value[0] - orig_value = "array of %s" %value # Check if type name for value is okay if isinstance(value, str): - if value == '**': - raise QAPIExprError(expr_info, - "%s uses '**' but did not request 'gen':false" - % source) - if not value in all_names: + if value not in all_names: raise QAPIExprError(expr_info, "%s uses unknown type '%s'" - % (source, orig_value)) + % (source, value)) if not all_names[value] in allow_metas: raise QAPIExprError(expr_info, "%s cannot use %s type '%s'" - % (source, all_names[value], orig_value)) + % (source, all_names[value], value)) return - # value is a dictionary, check that each member is okay - if not isinstance(value, OrderedDict): - raise QAPIExprError(expr_info, - "%s should be a dictionary" % source) if not allow_dict: raise QAPIExprError(expr_info, "%s should be a type name" % source) + + if not isinstance(value, OrderedDict): + raise QAPIExprError(expr_info, + "%s should be a dictionary or type name" % source) + + # value is a dictionary, check that each member is okay for (key, arg) in value.items(): check_name(expr_info, "Member of %s" % source, key, allow_optional=allow_optional) + if c_name(key, False) == 'u' or c_name(key, False).startswith('has_'): + raise QAPIExprError(expr_info, + "Member of %s uses reserved name '%s'" + % (source, key)) # Todo: allow dictionaries to represent default values of # an optional argument. check_type(expr_info, "Member '%s' of %s" % (key, source), arg, - allow_array=True, allow_star=allow_star, + allow_array=True, allow_metas=['built-in', 'union', 'alternate', 'struct', 'enum']) -def check_member_clash(expr_info, base_name, data, source = ""): - base = find_struct(base_name) - assert base - base_members = base['data'] - for key in data.keys(): - if key.startswith('*'): - key = key[1:] - if key in base_members or "*" + key in base_members: - raise QAPIExprError(expr_info, - "Member name '%s'%s clashes with base '%s'" - % (key, source, base_name)) - if base.get('base'): - check_member_clash(expr_info, base['base'], data, source) def check_command(expr, expr_info): name = expr['command'] - allow_star = expr.has_key('gen') check_type(expr_info, "'data' for command '%s'" % name, expr.get('data'), allow_dict=True, allow_optional=True, - allow_metas=['union', 'struct'], allow_star=allow_star) + allow_metas=['struct']) returns_meta = ['union', 'struct'] if name in returns_whitelist: returns_meta += ['built-in', 'alternate', 'enum'] check_type(expr_info, "'returns' for command '%s'" % name, - expr.get('returns'), allow_array=True, allow_dict=True, - allow_optional=True, allow_metas=returns_meta, - allow_star=allow_star) + expr.get('returns'), allow_array=True, + allow_optional=True, allow_metas=returns_meta) + def check_event(expr, expr_info): global events name = expr['event'] - params = expr.get('data') - if name.upper() == 'MAX': - raise QAPIExprError(expr_info, "Event name 'MAX' cannot be created") events.append(name) check_type(expr_info, "'data' for event '%s'" % name, expr.get('data'), allow_dict=True, allow_optional=True, - allow_metas=['union', 'struct']) + allow_metas=['struct']) + def check_union(expr, expr_info): name = expr['union'] base = expr.get('base') discriminator = expr.get('discriminator') members = expr['data'] - values = { 'MAX': '(automatic)' } - - # If the object has a member 'base', its value must name a struct, - # and there must be a discriminator. - if base is not None: - if discriminator is None: - raise QAPIExprError(expr_info, - "Union '%s' requires a discriminator to go " - "along with base" %name) # Two types of unions, determined by discriminator. # With no discriminator it is a simple union. if discriminator is None: enum_define = None - allow_metas=['built-in', 'union', 'alternate', 'struct', 'enum'] + allow_metas = ['built-in', 'union', 'alternate', 'struct', 'enum'] if base is not None: raise QAPIExprError(expr_info, "Simple union '%s' must not have a base" @@ -544,91 +563,78 @@ def check_union(expr, expr_info): # Else, it's a flat union. else: - # The object must have a string member 'base'. - if not isinstance(base, str): + # The object must have a string or dictionary 'base'. + check_type(expr_info, "'base' for union '%s'" % name, + base, allow_dict=True, allow_optional=True, + allow_metas=['struct']) + if not base: raise QAPIExprError(expr_info, - "Flat union '%s' must have a string base field" + "Flat union '%s' must have a base" % name) - base_fields = find_base_fields(base) - if not base_fields: - raise QAPIExprError(expr_info, - "Base '%s' is not a valid struct" - % base) + base_members = find_base_members(base) + assert base_members # The value of member 'discriminator' must name a non-optional # member of the base struct. check_name(expr_info, "Discriminator of flat union '%s'" % name, discriminator) - discriminator_type = base_fields.get(discriminator) + discriminator_type = base_members.get(discriminator) if not discriminator_type: raise QAPIExprError(expr_info, "Discriminator '%s' is not a member of base " "struct '%s'" % (discriminator, base)) enum_define = find_enum(discriminator_type) - allow_metas=['struct'] + allow_metas = ['struct'] # Do not allow string discriminator if not enum_define: raise QAPIExprError(expr_info, "Discriminator '%s' must be of enumeration " "type" % discriminator) - # Check every branch + # Check every branch; don't allow an empty union + if len(members) == 0: + raise QAPIExprError(expr_info, + "Union '%s' cannot have empty 'data'" % name) for (key, value) in members.items(): check_name(expr_info, "Member of union '%s'" % name, key) - # Each value must name a known type; furthermore, in flat unions, - # branches must be a struct with no overlapping member names + # Each value must name a known type check_type(expr_info, "Member '%s' of union '%s'" % (key, name), value, allow_array=not base, allow_metas=allow_metas) - if base: - branch_struct = find_struct(value) - assert branch_struct - check_member_clash(expr_info, base, branch_struct['data'], - " of branch '%s'" % key) # If the discriminator names an enum type, then all members # of 'data' must also be members of the enum type. if enum_define: - if not key in enum_define['enum_values']: + if key not in enum_define['enum_values']: raise QAPIExprError(expr_info, "Discriminator value '%s' is not found in " "enum '%s'" % (key, enum_define["enum_name"])) - # Otherwise, check for conflicts in the generated enum - else: - c_key = camel_to_upper(key) - if c_key in values: - raise QAPIExprError(expr_info, - "Union '%s' member '%s' clashes with '%s'" - % (name, key, values[c_key])) - values[c_key] = key def check_alternate(expr, expr_info): name = expr['alternate'] members = expr['data'] - values = { 'MAX': '(automatic)' } types_seen = {} - # Check every branch + # Check every branch; require at least two branches + if len(members) < 2: + raise QAPIExprError(expr_info, + "Alternate '%s' should have at least two branches " + "in 'data'" % name) for (key, value) in members.items(): check_name(expr_info, "Member of alternate '%s'" % name, key) - # Check for conflicts in the generated enum - c_key = camel_to_upper(key) - if c_key in values: - raise QAPIExprError(expr_info, - "Alternate '%s' member '%s' clashes with '%s'" - % (name, key, values[c_key])) - values[c_key] = key - # Ensure alternates have no type conflicts. check_type(expr_info, "Member '%s' of alternate '%s'" % (key, name), value, allow_metas=['built-in', 'union', 'struct', 'enum']) qtype = find_alternate_member_qtype(value) - assert qtype + if not qtype: + raise QAPIExprError(expr_info, + "Alternate '%s' member '%s' cannot use " + "type '%s'" % (name, key, value)) if qtype in types_seen: raise QAPIExprError(expr_info, "Alternate '%s' member '%s' can't " @@ -636,23 +642,22 @@ def check_alternate(expr, expr_info): % (name, key, types_seen[qtype])) types_seen[qtype] = key + def check_enum(expr, expr_info): name = expr['enum'] members = expr.get('data') - values = { 'MAX': '(automatic)' } + prefix = expr.get('prefix') if not isinstance(members, list): raise QAPIExprError(expr_info, "Enum '%s' requires an array for 'data'" % name) + if prefix is not None and not isinstance(prefix, str): + raise QAPIExprError(expr_info, + "Enum '%s' requires a string for 'prefix'" % name) for member in members: - check_name(expr_info, "Member of enum '%s'" %name, member, + check_name(expr_info, "Member of enum '%s'" % name, member, enum_member=True) - key = camel_to_upper(member) - if key in values: - raise QAPIExprError(expr_info, - "Enum '%s' member '%s' clashes with '%s'" - % (name, member, values[key])) - values[key] = member + def check_struct(expr, expr_info): name = expr['struct'] @@ -662,8 +667,7 @@ def check_struct(expr, expr_info): allow_dict=True, allow_optional=True) check_type(expr_info, "'base' for struct '%s'" % name, expr.get('base'), allow_metas=['struct']) - if expr.get('base'): - check_member_clash(expr_info, expr['base'], expr['data']) + def check_keys(expr_elem, meta, required, optional=[]): expr = expr_elem['expr'] @@ -672,22 +676,23 @@ def check_keys(expr_elem, meta, required, optional=[]): if not isinstance(name, str): raise QAPIExprError(info, "'%s' key must have a string value" % meta) - required = required + [ meta ] + required = required + [meta] for (key, value) in expr.items(): - if not key in required and not key in optional: + if key not in required and key not in optional: raise QAPIExprError(info, "Unknown key '%s' in %s '%s'" % (key, meta, name)) - if (key == 'gen' or key == 'success-response') and value != False: + if (key == 'gen' or key == 'success-response') and value is not False: raise QAPIExprError(info, "'%s' of %s '%s' should only use false value" % (key, meta, name)) for key in required: - if not expr.has_key(key): + if key not in expr: raise QAPIExprError(info, "Key '%s' is missing from %s '%s'" % (key, meta, name)) + def check_exprs(exprs): global all_names @@ -697,24 +702,24 @@ def check_exprs(exprs): for expr_elem in exprs: expr = expr_elem['expr'] info = expr_elem['info'] - if expr.has_key('enum'): - check_keys(expr_elem, 'enum', ['data']) + if 'enum' in expr: + check_keys(expr_elem, 'enum', ['data'], ['prefix']) add_enum(expr['enum'], info, expr['data']) - elif expr.has_key('union'): + elif 'union' in expr: check_keys(expr_elem, 'union', ['data'], ['base', 'discriminator']) add_union(expr, info) - elif expr.has_key('alternate'): + elif 'alternate' in expr: check_keys(expr_elem, 'alternate', ['data']) add_name(expr['alternate'], info, 'alternate') - elif expr.has_key('struct'): + elif 'struct' in expr: check_keys(expr_elem, 'struct', ['data'], ['base']) add_struct(expr, info) - elif expr.has_key('command'): + elif 'command' in expr: check_keys(expr_elem, 'command', [], ['data', 'returns', 'gen', 'success-response']) add_name(expr['command'], info, 'command') - elif expr.has_key('event'): + elif 'event' in expr: check_keys(expr_elem, 'event', [], ['data']) add_name(expr['event'], info, 'event') else: @@ -724,11 +729,11 @@ def check_exprs(exprs): # Try again for hidden UnionKind enum for expr_elem in exprs: expr = expr_elem['expr'] - if expr.has_key('union'): + if 'union' in expr: if not discriminator_find_enum_define(expr): add_enum('%sKind' % expr['union'], expr_elem['info'], implicit=True) - elif expr.has_key('alternate'): + elif 'alternate' in expr: add_enum('%sKind' % expr['alternate'], expr_elem['info'], implicit=True) @@ -737,51 +742,683 @@ def check_exprs(exprs): expr = expr_elem['expr'] info = expr_elem['info'] - if expr.has_key('enum'): + if 'enum' in expr: check_enum(expr, info) - elif expr.has_key('union'): + elif 'union' in expr: check_union(expr, info) - elif expr.has_key('alternate'): + elif 'alternate' in expr: check_alternate(expr, info) - elif expr.has_key('struct'): + elif 'struct' in expr: check_struct(expr, info) - elif expr.has_key('command'): + elif 'command' in expr: check_command(expr, info) - elif expr.has_key('event'): + elif 'event' in expr: check_event(expr, info) else: assert False, 'unexpected meta type' - return map(lambda expr_elem: expr_elem['expr'], exprs) + return exprs -def parse_schema(fname): - try: - schema = QAPISchema(open(fname, "r")) - return check_exprs(schema.exprs) - except (QAPISchemaError, QAPIExprError), e: - print >>sys.stderr, e - exit(1) # -# Code generation helpers +# Schema compiler frontend # -def parse_args(typeinfo): - if isinstance(typeinfo, str): - struct = find_struct(typeinfo) - assert struct != None - typeinfo = struct['data'] +class QAPISchemaEntity(object): + def __init__(self, name, info): + assert isinstance(name, str) + self.name = name + # For explicitly defined entities, info points to the (explicit) + # definition. For builtins (and their arrays), info is None. + # For implicitly defined entities, info points to a place that + # triggered the implicit definition (there may be more than one + # such place). + self.info = info + + def c_name(self): + return c_name(self.name) + + def check(self, schema): + pass + + def is_implicit(self): + return not self.info + + def visit(self, visitor): + pass + + +class QAPISchemaVisitor(object): + def visit_begin(self, schema): + pass + + def visit_end(self): + pass + + def visit_needed(self, entity): + # Default to visiting everything + return True + + def visit_builtin_type(self, name, info, json_type): + pass + + def visit_enum_type(self, name, info, values, prefix): + pass + + def visit_array_type(self, name, info, element_type): + pass + + def visit_object_type(self, name, info, base, members, variants): + pass + + def visit_object_type_flat(self, name, info, members, variants): + pass + + def visit_alternate_type(self, name, info, variants): + pass + + def visit_command(self, name, info, arg_type, ret_type, + gen, success_response): + pass - for member in typeinfo: - argname = member - argentry = typeinfo[member] + def visit_event(self, name, info, arg_type): + pass + + +class QAPISchemaType(QAPISchemaEntity): + # Return the C type for common use. + # For the types we commonly box, this is a pointer type. + def c_type(self): + pass + + # Return the C type to be used in a parameter list. + def c_param_type(self): + return self.c_type() + + # Return the C type to be used where we suppress boxing. + def c_unboxed_type(self): + return self.c_type() + + def json_type(self): + pass + + def alternate_qtype(self): + json2qtype = { + 'string': 'QTYPE_QSTRING', + 'number': 'QTYPE_QFLOAT', + 'int': 'QTYPE_QINT', + 'boolean': 'QTYPE_QBOOL', + 'object': 'QTYPE_QDICT' + } + return json2qtype.get(self.json_type()) + + +class QAPISchemaBuiltinType(QAPISchemaType): + def __init__(self, name, json_type, c_type): + QAPISchemaType.__init__(self, name, None) + assert not c_type or isinstance(c_type, str) + assert json_type in ('string', 'number', 'int', 'boolean', 'null', + 'value') + self._json_type_name = json_type + self._c_type_name = c_type + + def c_name(self): + return self.name + + def c_type(self): + return self._c_type_name + + def c_param_type(self): + if self.name == 'str': + return 'const ' + self._c_type_name + return self._c_type_name + + def json_type(self): + return self._json_type_name + + def visit(self, visitor): + visitor.visit_builtin_type(self.name, self.info, self.json_type()) + + +class QAPISchemaEnumType(QAPISchemaType): + def __init__(self, name, info, values, prefix): + QAPISchemaType.__init__(self, name, info) + for v in values: + assert isinstance(v, QAPISchemaMember) + v.set_owner(name) + assert prefix is None or isinstance(prefix, str) + self.values = values + self.prefix = prefix + + def check(self, schema): + seen = {} + for v in self.values: + v.check_clash(self.info, seen) + + def is_implicit(self): + # See QAPISchema._make_implicit_enum_type() + return self.name.endswith('Kind') + + def c_type(self): + return c_name(self.name) + + def member_names(self): + return [v.name for v in self.values] + + def json_type(self): + return 'string' + + def visit(self, visitor): + visitor.visit_enum_type(self.name, self.info, + self.member_names(), self.prefix) + + +class QAPISchemaArrayType(QAPISchemaType): + def __init__(self, name, info, element_type): + QAPISchemaType.__init__(self, name, info) + assert isinstance(element_type, str) + self._element_type_name = element_type + self.element_type = None + + def check(self, schema): + self.element_type = schema.lookup_type(self._element_type_name) + assert self.element_type + + def is_implicit(self): + return True + + def c_type(self): + return c_name(self.name) + pointer_suffix + + def json_type(self): + return 'array' + + def visit(self, visitor): + visitor.visit_array_type(self.name, self.info, self.element_type) + + +class QAPISchemaObjectType(QAPISchemaType): + def __init__(self, name, info, base, local_members, variants): + # struct has local_members, optional base, and no variants + # flat union has base, variants, and no local_members + # simple union has local_members, variants, and no base + QAPISchemaType.__init__(self, name, info) + assert base is None or isinstance(base, str) + for m in local_members: + assert isinstance(m, QAPISchemaObjectTypeMember) + m.set_owner(name) + if variants is not None: + assert isinstance(variants, QAPISchemaObjectTypeVariants) + variants.set_owner(name) + self._base_name = base + self.base = None + self.local_members = local_members + self.variants = variants + self.members = None + + def check(self, schema): + if self.members is False: # check for cycles + raise QAPIExprError(self.info, + "Object %s contains itself" % self.name) + if self.members: + return + self.members = False # mark as being checked + seen = OrderedDict() + if self._base_name: + self.base = schema.lookup_type(self._base_name) + assert isinstance(self.base, QAPISchemaObjectType) + self.base.check(schema) + self.base.check_clash(schema, self.info, seen) + for m in self.local_members: + m.check(schema) + m.check_clash(self.info, seen) + self.members = seen.values() + if self.variants: + self.variants.check(schema, seen) + assert self.variants.tag_member in self.members + self.variants.check_clash(schema, self.info, seen) + + # Check that the members of this type do not cause duplicate JSON members, + # and update seen to track the members seen so far. Report any errors + # on behalf of info, which is not necessarily self.info + def check_clash(self, schema, info, seen): + assert not self.variants # not implemented + for m in self.members: + m.check_clash(info, seen) + + def is_implicit(self): + # See QAPISchema._make_implicit_object_type(), as well as + # _def_predefineds() + return self.name.startswith('q_') + + def c_name(self): + return QAPISchemaType.c_name(self) + + def c_type(self): + assert not self.is_implicit() + return c_name(self.name) + pointer_suffix + + def c_unboxed_type(self): + return c_name(self.name) + + def json_type(self): + return 'object' + + def visit(self, visitor): + visitor.visit_object_type(self.name, self.info, + self.base, self.local_members, self.variants) + visitor.visit_object_type_flat(self.name, self.info, + self.members, self.variants) + + +class QAPISchemaMember(object): + role = 'member' + + def __init__(self, name): + assert isinstance(name, str) + self.name = name + self.owner = None + + def set_owner(self, name): + assert not self.owner + self.owner = name + + def check_clash(self, info, seen): + cname = c_name(self.name) + if cname.lower() != cname and self.owner not in case_whitelist: + raise QAPIExprError(info, + "%s should not use uppercase" % self.describe()) + if cname in seen: + raise QAPIExprError(info, + "%s collides with %s" + % (self.describe(), seen[cname].describe())) + seen[cname] = self + + def _pretty_owner(self): + owner = self.owner + if owner.startswith('q_obj_'): + # See QAPISchema._make_implicit_object_type() - reverse the + # mapping there to create a nice human-readable description + owner = owner[6:] + if owner.endswith('-arg'): + return '(parameter of %s)' % owner[:-4] + elif owner.endswith('-base'): + return '(base of %s)' % owner[:-5] + else: + assert owner.endswith('-wrapper') + # Unreachable and not implemented + assert False + if owner.endswith('Kind'): + # See QAPISchema._make_implicit_enum_type() + return '(branch of %s)' % owner[:-4] + return '(%s of %s)' % (self.role, owner) + + def describe(self): + return "'%s' %s" % (self.name, self._pretty_owner()) + + +class QAPISchemaObjectTypeMember(QAPISchemaMember): + def __init__(self, name, typ, optional): + QAPISchemaMember.__init__(self, name) + assert isinstance(typ, str) + assert isinstance(optional, bool) + self._type_name = typ + self.type = None + self.optional = optional + + def check(self, schema): + assert self.owner + self.type = schema.lookup_type(self._type_name) + assert self.type + + +class QAPISchemaObjectTypeVariants(object): + def __init__(self, tag_name, tag_member, variants): + # Flat unions pass tag_name but not tag_member. + # Simple unions and alternates pass tag_member but not tag_name. + # After check(), tag_member is always set, and tag_name remains + # a reliable witness of being used by a flat union. + assert bool(tag_member) != bool(tag_name) + assert (isinstance(tag_name, str) or + isinstance(tag_member, QAPISchemaObjectTypeMember)) + assert len(variants) > 0 + for v in variants: + assert isinstance(v, QAPISchemaObjectTypeVariant) + self.tag_name = tag_name + self.tag_member = tag_member + self.variants = variants + + def set_owner(self, name): + for v in self.variants: + v.set_owner(name) + + def check(self, schema, seen): + if not self.tag_member: # flat union + self.tag_member = seen[c_name(self.tag_name)] + assert self.tag_name == self.tag_member.name + assert isinstance(self.tag_member.type, QAPISchemaEnumType) + for v in self.variants: + v.check(schema) + # Union names must match enum values; alternate names are + # checked separately. Use 'seen' to tell the two apart. + if seen: + assert v.name in self.tag_member.type.member_names() + assert isinstance(v.type, QAPISchemaObjectType) + v.type.check(schema) + + def check_clash(self, schema, info, seen): + for v in self.variants: + # Reset seen map for each variant, since qapi names from one + # branch do not affect another branch + assert isinstance(v.type, QAPISchemaObjectType) + v.type.check_clash(schema, info, dict(seen)) + + +class QAPISchemaObjectTypeVariant(QAPISchemaObjectTypeMember): + role = 'branch' + + def __init__(self, name, typ): + QAPISchemaObjectTypeMember.__init__(self, name, typ, False) + + +class QAPISchemaAlternateType(QAPISchemaType): + def __init__(self, name, info, variants): + QAPISchemaType.__init__(self, name, info) + assert isinstance(variants, QAPISchemaObjectTypeVariants) + assert not variants.tag_name + variants.set_owner(name) + variants.tag_member.set_owner(self.name) + self.variants = variants + + def check(self, schema): + self.variants.tag_member.check(schema) + # Not calling self.variants.check_clash(), because there's nothing + # to clash with + self.variants.check(schema, {}) + # Alternate branch names have no relation to the tag enum values; + # so we have to check for potential name collisions ourselves. + seen = {} + for v in self.variants.variants: + v.check_clash(self.info, seen) + + def c_type(self): + return c_name(self.name) + pointer_suffix + + def json_type(self): + return 'value' + + def visit(self, visitor): + visitor.visit_alternate_type(self.name, self.info, self.variants) + + +class QAPISchemaCommand(QAPISchemaEntity): + def __init__(self, name, info, arg_type, ret_type, gen, success_response): + QAPISchemaEntity.__init__(self, name, info) + assert not arg_type or isinstance(arg_type, str) + assert not ret_type or isinstance(ret_type, str) + self._arg_type_name = arg_type + self.arg_type = None + self._ret_type_name = ret_type + self.ret_type = None + self.gen = gen + self.success_response = success_response + + def check(self, schema): + if self._arg_type_name: + self.arg_type = schema.lookup_type(self._arg_type_name) + assert isinstance(self.arg_type, QAPISchemaObjectType) + assert not self.arg_type.variants # not implemented + if self._ret_type_name: + self.ret_type = schema.lookup_type(self._ret_type_name) + assert isinstance(self.ret_type, QAPISchemaType) + + def visit(self, visitor): + visitor.visit_command(self.name, self.info, + self.arg_type, self.ret_type, + self.gen, self.success_response) + + +class QAPISchemaEvent(QAPISchemaEntity): + def __init__(self, name, info, arg_type): + QAPISchemaEntity.__init__(self, name, info) + assert not arg_type or isinstance(arg_type, str) + self._arg_type_name = arg_type + self.arg_type = None + + def check(self, schema): + if self._arg_type_name: + self.arg_type = schema.lookup_type(self._arg_type_name) + assert isinstance(self.arg_type, QAPISchemaObjectType) + assert not self.arg_type.variants # not implemented + + def visit(self, visitor): + visitor.visit_event(self.name, self.info, self.arg_type) + + +class QAPISchema(object): + def __init__(self, fname): + try: + self.exprs = check_exprs(QAPISchemaParser(open(fname, "r")).exprs) + self._entity_dict = {} + self._predefining = True + self._def_predefineds() + self._predefining = False + self._def_exprs() + self.check() + except (QAPISchemaError, QAPIExprError) as err: + print >>sys.stderr, err + exit(1) + + def _def_entity(self, ent): + # Only the predefined types are allowed to not have info + assert ent.info or self._predefining + assert ent.name not in self._entity_dict + self._entity_dict[ent.name] = ent + + def lookup_entity(self, name, typ=None): + ent = self._entity_dict.get(name) + if typ and not isinstance(ent, typ): + return None + return ent + + def lookup_type(self, name): + return self.lookup_entity(name, QAPISchemaType) + + def _def_builtin_type(self, name, json_type, c_type): + self._def_entity(QAPISchemaBuiltinType(name, json_type, c_type)) + # TODO As long as we have QAPI_TYPES_BUILTIN to share multiple + # qapi-types.h from a single .c, all arrays of builtins must be + # declared in the first file whether or not they are used. Nicer + # would be to use lazy instantiation, while figuring out how to + # avoid compilation issues with multiple qapi-types.h. + self._make_array_type(name, None) + + def _def_predefineds(self): + for t in [('str', 'string', 'char' + pointer_suffix), + ('number', 'number', 'double'), + ('int', 'int', 'int64_t'), + ('int8', 'int', 'int8_t'), + ('int16', 'int', 'int16_t'), + ('int32', 'int', 'int32_t'), + ('int64', 'int', 'int64_t'), + ('uint8', 'int', 'uint8_t'), + ('uint16', 'int', 'uint16_t'), + ('uint32', 'int', 'uint32_t'), + ('uint64', 'int', 'uint64_t'), + ('size', 'int', 'uint64_t'), + ('bool', 'boolean', 'bool'), + ('any', 'value', 'QObject' + pointer_suffix)]: + self._def_builtin_type(*t) + self.the_empty_object_type = QAPISchemaObjectType('q_empty', None, + None, [], None) + self._def_entity(self.the_empty_object_type) + qtype_values = self._make_enum_members(['none', 'qnull', 'qint', + 'qstring', 'qdict', 'qlist', + 'qfloat', 'qbool']) + self._def_entity(QAPISchemaEnumType('QType', None, qtype_values, + 'QTYPE')) + + def _make_enum_members(self, values): + return [QAPISchemaMember(v) for v in values] + + def _make_implicit_enum_type(self, name, info, values): + # See also QAPISchemaObjectTypeMember._pretty_owner() + name = name + 'Kind' # Use namespace reserved by add_name() + self._def_entity(QAPISchemaEnumType( + name, info, self._make_enum_members(values), None)) + return name + + def _make_array_type(self, element_type, info): + name = element_type + 'List' # Use namespace reserved by add_name() + if not self.lookup_type(name): + self._def_entity(QAPISchemaArrayType(name, info, element_type)) + return name + + def _make_implicit_object_type(self, name, info, role, members): + if not members: + return None + # See also QAPISchemaObjectTypeMember._pretty_owner() + name = 'q_obj_%s-%s' % (name, role) + if not self.lookup_entity(name, QAPISchemaObjectType): + self._def_entity(QAPISchemaObjectType(name, info, None, + members, None)) + return name + + def _def_enum_type(self, expr, info): + name = expr['enum'] + data = expr['data'] + prefix = expr.get('prefix') + self._def_entity(QAPISchemaEnumType( + name, info, self._make_enum_members(data), prefix)) + + def _make_member(self, name, typ, info): optional = False - if member.startswith('*'): - argname = member[1:] + if name.startswith('*'): + name = name[1:] optional = True - # Todo: allow argentry to be OrderedDict, for providing the - # value of an optional argument. - yield (argname, argentry, optional) + if isinstance(typ, list): + assert len(typ) == 1 + typ = self._make_array_type(typ[0], info) + return QAPISchemaObjectTypeMember(name, typ, optional) + + def _make_members(self, data, info): + return [self._make_member(key, value, info) + for (key, value) in data.iteritems()] + + def _def_struct_type(self, expr, info): + name = expr['struct'] + base = expr.get('base') + data = expr['data'] + self._def_entity(QAPISchemaObjectType(name, info, base, + self._make_members(data, info), + None)) + + def _make_variant(self, case, typ): + return QAPISchemaObjectTypeVariant(case, typ) + + def _make_simple_variant(self, case, typ, info): + if isinstance(typ, list): + assert len(typ) == 1 + typ = self._make_array_type(typ[0], info) + typ = self._make_implicit_object_type( + typ, info, 'wrapper', [self._make_member('data', typ, info)]) + return QAPISchemaObjectTypeVariant(case, typ) + + def _def_union_type(self, expr, info): + name = expr['union'] + data = expr['data'] + base = expr.get('base') + tag_name = expr.get('discriminator') + tag_member = None + if isinstance(base, dict): + base = (self._make_implicit_object_type( + name, info, 'base', self._make_members(base, info))) + if tag_name: + variants = [self._make_variant(key, value) + for (key, value) in data.iteritems()] + members = [] + else: + variants = [self._make_simple_variant(key, value, info) + for (key, value) in data.iteritems()] + typ = self._make_implicit_enum_type(name, info, + [v.name for v in variants]) + tag_member = QAPISchemaObjectTypeMember('type', typ, False) + members = [tag_member] + self._def_entity( + QAPISchemaObjectType(name, info, base, members, + QAPISchemaObjectTypeVariants(tag_name, + tag_member, + variants))) + + def _def_alternate_type(self, expr, info): + name = expr['alternate'] + data = expr['data'] + variants = [self._make_variant(key, value) + for (key, value) in data.iteritems()] + tag_member = QAPISchemaObjectTypeMember('type', 'QType', False) + self._def_entity( + QAPISchemaAlternateType(name, info, + QAPISchemaObjectTypeVariants(None, + tag_member, + variants))) + + def _def_command(self, expr, info): + name = expr['command'] + data = expr.get('data') + rets = expr.get('returns') + gen = expr.get('gen', True) + success_response = expr.get('success-response', True) + if isinstance(data, OrderedDict): + data = self._make_implicit_object_type( + name, info, 'arg', self._make_members(data, info)) + if isinstance(rets, list): + assert len(rets) == 1 + rets = self._make_array_type(rets[0], info) + self._def_entity(QAPISchemaCommand(name, info, data, rets, gen, + success_response)) + + def _def_event(self, expr, info): + name = expr['event'] + data = expr.get('data') + if isinstance(data, OrderedDict): + data = self._make_implicit_object_type( + name, info, 'arg', self._make_members(data, info)) + self._def_entity(QAPISchemaEvent(name, info, data)) + + def _def_exprs(self): + for expr_elem in self.exprs: + expr = expr_elem['expr'] + info = expr_elem['info'] + if 'enum' in expr: + self._def_enum_type(expr, info) + elif 'struct' in expr: + self._def_struct_type(expr, info) + elif 'union' in expr: + self._def_union_type(expr, info) + elif 'alternate' in expr: + self._def_alternate_type(expr, info) + elif 'command' in expr: + self._def_command(expr, info) + elif 'event' in expr: + self._def_event(expr, info) + else: + assert False + + def check(self): + for ent in self._entity_dict.values(): + ent.check(self) + + def visit(self, visitor): + visitor.visit_begin(self) + for (name, entity) in sorted(self._entity_dict.items()): + if visitor.visit_needed(entity): + entity.visit(visitor) + visitor.visit_end() + + +# +# Code generation helpers +# def camel_case(name): new_name = '' @@ -796,6 +1433,7 @@ def camel_case(name): new_name += ch.lower() return new_name + # ENUMName -> ENUM_NAME, EnumName1 -> ENUM_NAME1 # ENUM_NAME -> ENUM_NAME, ENUM_NAME1 -> ENUM_NAME1, ENUM_Name2 -> ENUM_NAME2 # ENUM24_Name -> ENUM24_NAME @@ -810,19 +1448,22 @@ def camel_to_upper(value): c = c_fun_str[i] # When c is upper and no "_" appears before, do more checks if c.isupper() and (i > 0) and c_fun_str[i - 1] != "_": - # Case 1: next string is lower - # Case 2: previous string is digit - if (i < (l - 1) and c_fun_str[i + 1].islower()) or \ - c_fun_str[i - 1].isdigit(): + if i < l - 1 and c_fun_str[i + 1].islower(): + new_name += '_' + elif c_fun_str[i - 1].isdigit(): new_name += '_' new_name += c return new_name.lstrip('_').upper() -def c_enum_const(type_name, const_name): - return camel_to_upper(type_name + '_' + const_name) + +def c_enum_const(type_name, const_name, prefix=None): + if prefix is not None: + type_name = prefix + return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper() c_name_trans = string.maketrans('.-', '__') + # Map @name to a valid C identifier. # If @protect, avoid returning certain ticklish identifiers (like # C keywords) by prepending "q_". @@ -835,15 +1476,16 @@ c_name_trans = string.maketrans('.-', '__') def c_name(name, protect=True): # ANSI X3J11/88-090, 3.1.1 c89_words = set(['auto', 'break', 'case', 'char', 'const', 'continue', - 'default', 'do', 'double', 'else', 'enum', 'extern', 'float', - 'for', 'goto', 'if', 'int', 'long', 'register', 'return', - 'short', 'signed', 'sizeof', 'static', 'struct', 'switch', - 'typedef', 'union', 'unsigned', 'void', 'volatile', 'while']) + 'default', 'do', 'double', 'else', 'enum', 'extern', + 'float', 'for', 'goto', 'if', 'int', 'long', 'register', + 'return', 'short', 'signed', 'sizeof', 'static', + 'struct', 'switch', 'typedef', 'union', 'unsigned', + 'void', 'volatile', 'while']) # ISO/IEC 9899:1999, 6.4.1 c99_words = set(['inline', 'restrict', '_Bool', '_Complex', '_Imaginary']) # ISO/IEC 9899:2011, 6.4.1 - c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic', '_Noreturn', - '_Static_assert', '_Thread_local']) + c11_words = set(['_Alignas', '_Alignof', '_Atomic', '_Generic', + '_Noreturn', '_Static_assert', '_Thread_local']) # GCC http://gcc.gnu.org/onlinedocs/gcc-4.7.1/gcc/C-Extensions.html # excluding _.* gcc_words = set(['asm', 'typeof']) @@ -858,109 +1500,58 @@ def c_name(name, protect=True): 'and', 'and_eq', 'bitand', 'bitor', 'compl', 'not', 'not_eq', 'or', 'or_eq', 'xor', 'xor_eq']) # namespace pollution: - polluted_words = set(['unix', 'errno']) - if protect and (name in c89_words | c99_words | c11_words | gcc_words | cpp_words | polluted_words): + polluted_words = set(['unix', 'errno', 'mips', 'sparc']) + name = name.translate(c_name_trans) + if protect and (name in c89_words | c99_words | c11_words | gcc_words + | cpp_words | polluted_words): return "q_" + name - return name.translate(c_name_trans) - -# Map type @name to the C typedef name for the list form. -# -# ['Name'] -> 'NameList', ['x-Foo'] -> 'x_FooList', ['int'] -> 'intList' -def c_list_type(name): - return type_name(name) + 'List' - -# Map type @value to the C typedef form. -# -# Used for converting 'type' from a 'member':'type' qapi definition -# into the alphanumeric portion of the type for a generated C parameter, -# as well as generated C function names. See c_type() for the rest of -# the conversion such as adding '*' on pointer types. -# 'int' -> 'int', '[x-Foo]' -> 'x_FooList', '__a.b_c' -> '__a_b_c' -def type_name(value): - if type(value) == list: - return c_list_type(value[0]) - if value in builtin_types.keys(): - return value - return c_name(value) + return name eatspace = '\033EATSPACE.' pointer_suffix = ' *' + eatspace -# Map type @name to its C type expression. -# If @is_param, const-qualify the string type. -# -# This function is used for computing the full C type of 'member':'name'. -# A special suffix is added in c_type() for pointer types, and it's -# stripped in mcgen(). So please notice this when you check the return -# value of c_type() outside mcgen(). -def c_type(value, is_param=False): - if value == 'str': - if is_param: - return 'const char' + pointer_suffix - return 'char' + pointer_suffix - - elif value == 'int': - return 'int64_t' - elif (value == 'int8' or value == 'int16' or value == 'int32' or - value == 'int64' or value == 'uint8' or value == 'uint16' or - value == 'uint32' or value == 'uint64'): - return value + '_t' - elif value == 'size': - return 'uint64_t' - elif value == 'bool': - return 'bool' - elif value == 'number': - return 'double' - elif type(value) == list: - return c_list_type(value[0]) + pointer_suffix - elif is_enum(value): - return c_name(value) - elif value == None: - return 'void' - elif value in events: - return camel_case(value) + 'Event' + pointer_suffix - else: - # complex type name - assert isinstance(value, str) and value != "" - return c_name(value) + pointer_suffix - -def is_c_ptr(value): - return c_type(value).endswith(pointer_suffix) def genindent(count): ret = "" - for i in range(count): + for _ in range(count): ret += " " return ret indent_level = 0 + def push_indent(indent_amount=4): global indent_level indent_level += indent_amount + def pop_indent(indent_amount=4): global indent_level indent_level -= indent_amount + +# Generate @code with @kwds interpolated. +# Obey indent_level, and strip eatspace. def cgen(code, **kwds): - indent = genindent(indent_level) - lines = code.split('\n') - lines = map(lambda x: indent + x, lines) - return '\n'.join(lines) % kwds + '\n' + raw = code % kwds + if indent_level: + indent = genindent(indent_level) + # re.subn() lacks flags support before Python 2.7, use re.compile() + raw = re.subn(re.compile("^.", re.MULTILINE), + indent + r'\g<0>', raw) + raw = raw[0] + return re.sub(re.escape(eatspace) + ' *', '', raw) + def mcgen(code, **kwds): - raw = cgen('\n'.join(code.split('\n')[1:-1]), **kwds) - return re.sub(re.escape(eatspace) + ' *', '', raw) + if code[0] == '\n': + code = code[1:] + return cgen(code, **kwds) -def basename(filename): - return filename.split("/")[-1] def guardname(filename): - guard = basename(filename).rsplit(".", 1)[0] - for substr in [".", " ", "-"]: - guard = guard.replace(substr, "_") - return guard.upper() + '_H' + return c_name(filename, protect=False).upper() + def guardstart(name): return mcgen(''' @@ -971,6 +1562,7 @@ def guardstart(name): ''', name=guardname(name)) + def guardend(name): return mcgen(''' @@ -979,18 +1571,99 @@ def guardend(name): ''', name=guardname(name)) + +def gen_enum_lookup(name, values, prefix=None): + ret = mcgen(''' + +const char *const %(c_name)s_lookup[] = { +''', + c_name=c_name(name)) + for value in values: + index = c_enum_const(name, value, prefix) + ret += mcgen(''' + [%(index)s] = "%(value)s", +''', + index=index, value=value) + + max_index = c_enum_const(name, '_MAX', prefix) + ret += mcgen(''' + [%(max_index)s] = NULL, +}; +''', + max_index=max_index) + return ret + + +def gen_enum(name, values, prefix=None): + # append automatically generated _MAX value + enum_values = values + ['_MAX'] + + ret = mcgen(''' + +typedef enum %(c_name)s { +''', + c_name=c_name(name)) + + i = 0 + for value in enum_values: + ret += mcgen(''' + %(c_enum)s = %(i)d, +''', + c_enum=c_enum_const(name, value, prefix), + i=i) + i += 1 + + ret += mcgen(''' +} %(c_name)s; +''', + c_name=c_name(name)) + + ret += mcgen(''' + +extern const char *const %(c_name)s_lookup[]; +''', + c_name=c_name(name)) + return ret + + +def gen_params(arg_type, extra): + if not arg_type: + return extra + assert not arg_type.variants + ret = '' + sep = '' + for memb in arg_type.members: + ret += sep + sep = ', ' + if memb.optional: + ret += 'bool has_%s, ' % c_name(memb.name) + ret += '%s %s' % (memb.type.c_param_type(), c_name(memb.name)) + if extra: + ret += sep + extra + return ret + + +def gen_err_check(): + return mcgen(''' + if (err) { + goto out; + } +''') + + # # Common command line parsing # -def parse_command_line(extra_options = "", extra_long_options = []): + +def parse_command_line(extra_options="", extra_long_options=[]): try: opts, args = getopt.gnu_getopt(sys.argv[1:], "chp:o:" + extra_options, ["source", "header", "prefix=", "output-dir="] + extra_long_options) - except getopt.GetoptError, err: + except getopt.GetoptError as err: print >>sys.stderr, "%s: %s" % (sys.argv[0], str(err)) sys.exit(1) @@ -1003,6 +1676,12 @@ def parse_command_line(extra_options = "", extra_long_options = []): for oa in opts: o, a = oa if o in ("-p", "--prefix"): + match = re.match('([A-Za-z_.-][A-Za-z0-9_.-]*)?', a) + if match.end() != len(a): + print >>sys.stderr, \ + "%s: 'funny character '%s' in argument of --prefix" \ + % (sys.argv[0], a[match.end()]) + sys.exit(1) prefix = a elif o in ("-o", "--output-dir"): output_dir = a + "/" @@ -1028,16 +1707,19 @@ def parse_command_line(extra_options = "", extra_long_options = []): # Generate output files with boilerplate # + def open_output(output_dir, do_c, do_h, prefix, c_file, h_file, c_comment, h_comment): + guard = guardname(prefix + h_file) c_file = output_dir + prefix + c_file h_file = output_dir + prefix + h_file - try: - os.makedirs(output_dir) - except os.error, e: - if e.errno != errno.EEXIST: - raise + if output_dir: + try: + os.makedirs(output_dir) + except os.error as e: + if e.errno != errno.EEXIST: + raise def maybe_open(really, name, opt): if really: @@ -1053,7 +1735,7 @@ def open_output(output_dir, do_c, do_h, prefix, c_file, h_file, /* AUTOMATICALLY GENERATED, DO NOT MODIFY */ %(comment)s ''', - comment = c_comment)) + comment=c_comment)) fdecl.write(mcgen(''' /* AUTOMATICALLY GENERATED, DO NOT MODIFY */ @@ -1062,10 +1744,11 @@ def open_output(output_dir, do_c, do_h, prefix, c_file, h_file, #define %(guard)s ''', - comment = h_comment, guard = guardname(h_file))) + comment=h_comment, guard=guard)) return (fdef, fdecl) + def close_output(fdef, fdecl): fdecl.write(''' #endif diff --git a/qemu/scripts/qemu-gdb.py b/qemu/scripts/qemu-gdb.py index 6c7f4fbe5..b3f8e04f7 100644 --- a/qemu/scripts/qemu-gdb.py +++ b/qemu/scripts/qemu-gdb.py @@ -13,73 +13,20 @@ # Contributions after 2012-01-13 are licensed under the terms of the # GNU GPL, version 2 or (at your option) any later version. +# Usage: +# At the (gdb) prompt, type "source scripts/qemu-gdb.py". +# "help qemu" should then list the supported QEMU debug support commands. import gdb -def isnull(ptr): - return ptr == gdb.Value(0).cast(ptr.type) +import os, sys -def int128(p): - return long(p['lo']) + (long(p['hi']) << 64) +# Annoyingly, gdb doesn't put the directory of scripts onto the +# module search path. Do it manually. -def get_fs_base(): - '''Fetch %fs base value using arch_prctl(ARCH_GET_FS)''' - # %rsp - 120 is scratch space according to the SystemV ABI - old = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)') - gdb.execute('call arch_prctl(0x1003, $rsp - 120)', False, True) - fs_base = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)') - gdb.execute('set *(uint64_t*)($rsp - 120) = %s' % old, False, True) - return fs_base +sys.path.append(os.path.dirname(__file__)) -def get_glibc_pointer_guard(): - '''Fetch glibc pointer guard value''' - fs_base = get_fs_base() - return gdb.parse_and_eval('*(uint64_t*)((uint64_t)%s + 0x30)' % fs_base) - -def glibc_ptr_demangle(val, pointer_guard): - '''Undo effect of glibc's PTR_MANGLE()''' - return gdb.parse_and_eval('(((uint64_t)%s >> 0x11) | ((uint64_t)%s << (64 - 0x11))) ^ (uint64_t)%s' % (val, val, pointer_guard)) - -def bt_jmpbuf(jmpbuf): - '''Backtrace a jmpbuf''' - JB_RBX = 0 - JB_RBP = 1 - JB_R12 = 2 - JB_R13 = 3 - JB_R14 = 4 - JB_R15 = 5 - JB_RSP = 6 - JB_PC = 7 - - old_rbx = gdb.parse_and_eval('(uint64_t)$rbx') - old_rbp = gdb.parse_and_eval('(uint64_t)$rbp') - old_rsp = gdb.parse_and_eval('(uint64_t)$rsp') - old_r12 = gdb.parse_and_eval('(uint64_t)$r12') - old_r13 = gdb.parse_and_eval('(uint64_t)$r13') - old_r14 = gdb.parse_and_eval('(uint64_t)$r14') - old_r15 = gdb.parse_and_eval('(uint64_t)$r15') - old_rip = gdb.parse_and_eval('(uint64_t)$rip') - - pointer_guard = get_glibc_pointer_guard() - gdb.execute('set $rbx = %s' % jmpbuf[JB_RBX]) - gdb.execute('set $rbp = %s' % glibc_ptr_demangle(jmpbuf[JB_RBP], pointer_guard)) - gdb.execute('set $rsp = %s' % glibc_ptr_demangle(jmpbuf[JB_RSP], pointer_guard)) - gdb.execute('set $r12 = %s' % jmpbuf[JB_R12]) - gdb.execute('set $r13 = %s' % jmpbuf[JB_R13]) - gdb.execute('set $r14 = %s' % jmpbuf[JB_R14]) - gdb.execute('set $r15 = %s' % jmpbuf[JB_R15]) - gdb.execute('set $rip = %s' % glibc_ptr_demangle(jmpbuf[JB_PC], pointer_guard)) - - gdb.execute('bt') - - gdb.execute('set $rbx = %s' % old_rbx) - gdb.execute('set $rbp = %s' % old_rbp) - gdb.execute('set $rsp = %s' % old_rsp) - gdb.execute('set $r12 = %s' % old_r12) - gdb.execute('set $r13 = %s' % old_r13) - gdb.execute('set $r14 = %s' % old_r14) - gdb.execute('set $r15 = %s' % old_r15) - gdb.execute('set $rip = %s' % old_rip) +from qemugdb import aio, mtree, coroutine class QemuCommand(gdb.Command): '''Prefix for QEMU debug support commands''' @@ -87,78 +34,14 @@ class QemuCommand(gdb.Command): gdb.Command.__init__(self, 'qemu', gdb.COMMAND_DATA, gdb.COMPLETE_NONE, True) -class CoroutineCommand(gdb.Command): - '''Display coroutine backtrace''' - def __init__(self): - gdb.Command.__init__(self, 'qemu coroutine', gdb.COMMAND_DATA, - gdb.COMPLETE_NONE) - - def invoke(self, arg, from_tty): - argv = gdb.string_to_argv(arg) - if len(argv) != 1: - gdb.write('usage: qemu coroutine <coroutine-pointer>\n') - return - - coroutine_pointer = gdb.parse_and_eval(argv[0]).cast(gdb.lookup_type('CoroutineUContext').pointer()) - bt_jmpbuf(coroutine_pointer['env']['__jmpbuf']) +QemuCommand() +coroutine.CoroutineCommand() +mtree.MtreeCommand() +aio.HandlersCommand() -class MtreeCommand(gdb.Command): - '''Display the memory tree hierarchy''' - def __init__(self): - gdb.Command.__init__(self, 'qemu mtree', gdb.COMMAND_DATA, - gdb.COMPLETE_NONE) - self.queue = [] - def invoke(self, arg, from_tty): - self.seen = set() - self.queue_root('address_space_memory') - self.queue_root('address_space_io') - self.process_queue() - def queue_root(self, varname): - ptr = gdb.parse_and_eval(varname)['root'] - self.queue.append(ptr) - def process_queue(self): - while self.queue: - ptr = self.queue.pop(0) - if long(ptr) in self.seen: - continue - self.print_item(ptr) - def print_item(self, ptr, offset = gdb.Value(0), level = 0): - self.seen.add(long(ptr)) - addr = ptr['addr'] - addr += offset - size = int128(ptr['size']) - alias = ptr['alias'] - klass = '' - if not isnull(alias): - klass = ' (alias)' - elif not isnull(ptr['ops']): - klass = ' (I/O)' - elif bool(ptr['ram']): - klass = ' (RAM)' - gdb.write('%s%016x-%016x %s%s (@ %s)\n' - % (' ' * level, - long(addr), - long(addr + (size - 1)), - ptr['name'].string(), - klass, - ptr, - ), - gdb.STDOUT) - if not isnull(alias): - gdb.write('%s alias: %s@%016x (@ %s)\n' % - (' ' * level, - alias['name'].string(), - ptr['alias_offset'], - alias, - ), - gdb.STDOUT) - self.queue.append(alias) - subregion = ptr['subregions']['tqh_first'] - level += 1 - while not isnull(subregion): - self.print_item(subregion, addr, level) - subregion = subregion['subregions_link']['tqe_next'] +coroutine.CoroutineSPFunction() +coroutine.CoroutinePCFunction() -QemuCommand() -CoroutineCommand() -MtreeCommand() +# Default to silently passing through SIGUSR1, because QEMU sends it +# to itself a lot. +gdb.execute('handle SIGUSR1 pass noprint nostop') diff --git a/qemu/scripts/qemugdb/__init__.py b/qemu/scripts/qemugdb/__init__.py new file mode 100644 index 000000000..969f552b2 --- /dev/null +++ b/qemu/scripts/qemugdb/__init__.py @@ -0,0 +1,28 @@ +#!/usr/bin/python + +# GDB debugging support +# +# Copyright (c) 2015 Linaro Ltd +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, see +# <http://www.gnu.org/licenses/gpl-2.0.html> +# + +# We don't need to do anything in our init file currently. + +""" +Support routines for debugging QEMU under GDB +""" + +__license__ = "GPL version 2 or (at your option) any later version" diff --git a/qemu/scripts/qemugdb/aio.py b/qemu/scripts/qemugdb/aio.py new file mode 100644 index 000000000..2ba00c444 --- /dev/null +++ b/qemu/scripts/qemugdb/aio.py @@ -0,0 +1,58 @@ +#!/usr/bin/python + +# GDB debugging support: aio/iohandler debug +# +# Copyright (c) 2015 Red Hat, Inc. +# +# Author: Dr. David Alan Gilbert <dgilbert@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2 or +# later. See the COPYING file in the top-level directory. +# + +import gdb +from qemugdb import coroutine + +def isnull(ptr): + return ptr == gdb.Value(0).cast(ptr.type) + +def dump_aiocontext(context, verbose): + '''Display a dump and backtrace for an aiocontext''' + cur = context['aio_handlers']['lh_first'] + # Get pointers to functions we're going to process specially + sym_fd_coroutine_enter = gdb.parse_and_eval('fd_coroutine_enter') + + while not isnull(cur): + entry = cur.dereference() + gdb.write('----\n%s\n' % entry) + if verbose and cur['io_read'] == sym_fd_coroutine_enter: + coptr = (cur['opaque'].cast(gdb.lookup_type('FDYieldUntilData').pointer()))['co'] + coptr = coptr.cast(gdb.lookup_type('CoroutineUContext').pointer()) + coroutine.bt_jmpbuf(coptr['env']['__jmpbuf']) + cur = cur['node']['le_next']; + + gdb.write('----\n') + +class HandlersCommand(gdb.Command): + '''Display aio handlers''' + def __init__(self): + gdb.Command.__init__(self, 'qemu handlers', gdb.COMMAND_DATA, + gdb.COMPLETE_NONE) + + def invoke(self, arg, from_tty): + verbose = False + argv = gdb.string_to_argv(arg) + + if len(argv) > 0 and argv[0] == '--verbose': + verbose = True + argv.pop(0) + + if len(argv) > 1: + gdb.write('usage: qemu handlers [--verbose] [handler]\n') + return + + if len(argv) == 1: + handlers_name = argv[0] + else: + handlers_name = 'qemu_aio_context' + dump_aiocontext(gdb.parse_and_eval(handlers_name), verbose) diff --git a/qemu/scripts/qemugdb/coroutine.py b/qemu/scripts/qemugdb/coroutine.py new file mode 100644 index 000000000..ab699794a --- /dev/null +++ b/qemu/scripts/qemugdb/coroutine.py @@ -0,0 +1,119 @@ +#!/usr/bin/python + +# GDB debugging support +# +# Copyright 2012 Red Hat, Inc. and/or its affiliates +# +# Authors: +# Avi Kivity <avi@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2. See +# the COPYING file in the top-level directory. +# +# Contributions after 2012-01-13 are licensed under the terms of the +# GNU GPL, version 2 or (at your option) any later version. + +import gdb + +VOID_PTR = gdb.lookup_type('void').pointer() + +def get_fs_base(): + '''Fetch %fs base value using arch_prctl(ARCH_GET_FS). This is + pthread_self().''' + # %rsp - 120 is scratch space according to the SystemV ABI + old = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)') + gdb.execute('call arch_prctl(0x1003, $rsp - 120)', False, True) + fs_base = gdb.parse_and_eval('*(uint64_t*)($rsp - 120)') + gdb.execute('set *(uint64_t*)($rsp - 120) = %s' % old, False, True) + return fs_base + +def pthread_self(): + '''Fetch pthread_self() from the glibc start_thread function.''' + f = gdb.newest_frame() + while f.name() != 'start_thread': + f = f.older() + if f is None: + return get_fs_base() + + try: + return f.read_var("arg") + except ValueError: + return get_fs_base() + +def get_glibc_pointer_guard(): + '''Fetch glibc pointer guard value''' + fs_base = pthread_self() + return gdb.parse_and_eval('*(uint64_t*)((uint64_t)%s + 0x30)' % fs_base) + +def glibc_ptr_demangle(val, pointer_guard): + '''Undo effect of glibc's PTR_MANGLE()''' + return gdb.parse_and_eval('(((uint64_t)%s >> 0x11) | ((uint64_t)%s << (64 - 0x11))) ^ (uint64_t)%s' % (val, val, pointer_guard)) + +def get_jmpbuf_regs(jmpbuf): + JB_RBX = 0 + JB_RBP = 1 + JB_R12 = 2 + JB_R13 = 3 + JB_R14 = 4 + JB_R15 = 5 + JB_RSP = 6 + JB_PC = 7 + + pointer_guard = get_glibc_pointer_guard() + return {'rbx': jmpbuf[JB_RBX], + 'rbp': glibc_ptr_demangle(jmpbuf[JB_RBP], pointer_guard), + 'rsp': glibc_ptr_demangle(jmpbuf[JB_RSP], pointer_guard), + 'r12': jmpbuf[JB_R12], + 'r13': jmpbuf[JB_R13], + 'r14': jmpbuf[JB_R14], + 'r15': jmpbuf[JB_R15], + 'rip': glibc_ptr_demangle(jmpbuf[JB_PC], pointer_guard) } + +def bt_jmpbuf(jmpbuf): + '''Backtrace a jmpbuf''' + regs = get_jmpbuf_regs(jmpbuf) + old = dict() + + for i in regs: + old[i] = gdb.parse_and_eval('(uint64_t)$%s' % i) + + for i in regs: + gdb.execute('set $%s = %s' % (i, regs[i])) + + gdb.execute('bt') + + for i in regs: + gdb.execute('set $%s = %s' % (i, old[i])) + +def coroutine_to_jmpbuf(co): + coroutine_pointer = co.cast(gdb.lookup_type('CoroutineUContext').pointer()) + return coroutine_pointer['env']['__jmpbuf'] + + +class CoroutineCommand(gdb.Command): + '''Display coroutine backtrace''' + def __init__(self): + gdb.Command.__init__(self, 'qemu coroutine', gdb.COMMAND_DATA, + gdb.COMPLETE_NONE) + + def invoke(self, arg, from_tty): + argv = gdb.string_to_argv(arg) + if len(argv) != 1: + gdb.write('usage: qemu coroutine <coroutine-pointer>\n') + return + + bt_jmpbuf(coroutine_to_jmpbuf(gdb.parse_and_eval(argv[0]))) + +class CoroutineSPFunction(gdb.Function): + def __init__(self): + gdb.Function.__init__(self, 'qemu_coroutine_sp') + + def invoke(self, addr): + return get_jmpbuf_regs(coroutine_to_jmpbuf(addr))['rsp'].cast(VOID_PTR) + +class CoroutinePCFunction(gdb.Function): + def __init__(self): + gdb.Function.__init__(self, 'qemu_coroutine_pc') + + def invoke(self, addr): + return get_jmpbuf_regs(coroutine_to_jmpbuf(addr))['rip'].cast(VOID_PTR) diff --git a/qemu/scripts/qemugdb/mtree.py b/qemu/scripts/qemugdb/mtree.py new file mode 100644 index 000000000..cc8131c2e --- /dev/null +++ b/qemu/scripts/qemugdb/mtree.py @@ -0,0 +1,82 @@ +#!/usr/bin/python + +# GDB debugging support +# +# Copyright 2012 Red Hat, Inc. and/or its affiliates +# +# Authors: +# Avi Kivity <avi@redhat.com> +# +# This work is licensed under the terms of the GNU GPL, version 2. See +# the COPYING file in the top-level directory. +# +# Contributions after 2012-01-13 are licensed under the terms of the +# GNU GPL, version 2 or (at your option) any later version. + +# 'qemu mtree' -- display the memory hierarchy + +import gdb + +def isnull(ptr): + return ptr == gdb.Value(0).cast(ptr.type) + +def int128(p): + return int(p['lo']) + (int(p['hi']) << 64) + +class MtreeCommand(gdb.Command): + '''Display the memory tree hierarchy''' + def __init__(self): + gdb.Command.__init__(self, 'qemu mtree', gdb.COMMAND_DATA, + gdb.COMPLETE_NONE) + self.queue = [] + def invoke(self, arg, from_tty): + self.seen = set() + self.queue_root('address_space_memory') + self.queue_root('address_space_io') + self.process_queue() + def queue_root(self, varname): + ptr = gdb.parse_and_eval(varname)['root'] + self.queue.append(ptr) + def process_queue(self): + while self.queue: + ptr = self.queue.pop(0) + if int(ptr) in self.seen: + continue + self.print_item(ptr) + def print_item(self, ptr, offset = gdb.Value(0), level = 0): + self.seen.add(int(ptr)) + addr = ptr['addr'] + addr += offset + size = int128(ptr['size']) + alias = ptr['alias'] + klass = '' + if not isnull(alias): + klass = ' (alias)' + elif not isnull(ptr['ops']): + klass = ' (I/O)' + elif bool(ptr['ram']): + klass = ' (RAM)' + gdb.write('%s%016x-%016x %s%s (@ %s)\n' + % (' ' * level, + int(addr), + int(addr + (size - 1)), + ptr['name'].string(), + klass, + ptr, + ), + gdb.STDOUT) + if not isnull(alias): + gdb.write('%s alias: %s@%016x (@ %s)\n' % + (' ' * level, + alias['name'].string(), + ptr['alias_offset'], + alias, + ), + gdb.STDOUT) + self.queue.append(alias) + subregion = ptr['subregions']['tqh_first'] + level += 1 + while not isnull(subregion): + self.print_item(subregion, addr, level) + subregion = subregion['subregions_link']['tqe_next'] + diff --git a/qemu/scripts/qmp/qemu-ga-client b/qemu/scripts/qmp/qemu-ga-client index 9908f2109..fd056056f 100755 --- a/qemu/scripts/qmp/qemu-ga-client +++ b/qemu/scripts/qmp/qemu-ga-client @@ -259,7 +259,7 @@ def main(address, cmd, args): try: client = QemuGuestAgentClient(address) - except QemuGuestAgent.error, e: + except QemuGuestAgent.error as e: import errno print(e) diff --git a/qemu/scripts/qmp/qmp b/qemu/scripts/qmp/qmp index 1db3c7ffe..514b539a6 100755 --- a/qemu/scripts/qmp/qmp +++ b/qemu/scripts/qmp/qmp @@ -91,8 +91,8 @@ def main(args): try: os.environ['QMP_PATH'] = path os.execvp(fullcmd, [fullcmd] + args) - except OSError, (errno, msg): - if errno == 2: + except OSError as exc: + if exc.errno == 2: print 'Command "%s" not found.' % (fullcmd) return 1 raise diff --git a/qemu/scripts/qmp/qmp-shell b/qemu/scripts/qmp/qmp-shell index 65280d29d..0373b24b2 100755 --- a/qemu/scripts/qmp/qmp-shell +++ b/qemu/scripts/qmp/qmp-shell @@ -29,13 +29,47 @@ # (QEMU) device_add driver=e1000 id=net1 # {u'return': {}} # (QEMU) +# +# key=value pairs also support Python or JSON object literal subset notations, +# without spaces. Dictionaries/objects {} are supported as are arrays []. +# +# example-command arg-name1={'key':'value','obj'={'prop':"value"}} +# +# Both JSON and Python formatting should work, including both styles of +# string literal quotes. Both paradigms of literal values should work, +# including null/true/false for JSON and None/True/False for Python. +# +# +# Transactions have the following multi-line format: +# +# transaction( +# action-name1 [ arg-name1=arg1 ] ... [arg-nameN=argN ] +# ... +# action-nameN [ arg-name1=arg1 ] ... [arg-nameN=argN ] +# ) +# +# One line transactions are also supported: +# +# transaction( action-name1 ... ) +# +# For example: +# +# (QEMU) transaction( +# TRANS> block-dirty-bitmap-add node=drive0 name=bitmap1 +# TRANS> block-dirty-bitmap-clear node=drive0 name=bitmap0 +# TRANS> ) +# {"return": {}} +# (QEMU) +# +# Use the -v and -p options to activate the verbose and pretty-print options, +# which will echo back the properly formatted JSON-compliant QMP that is being +# sent to QEMU, which is useful for debugging and documentation generation. import qmp import json import ast import readline import sys -import pprint class QMPCompleter(list): def complete(self, text, state): @@ -68,11 +102,11 @@ class FuzzyJSON(ast.NodeTransformer): # TODO: QMPShell's interface is a bit ugly (eg. _fill_completion() and # _execute_cmd()). Let's design a better one. class QMPShell(qmp.QEMUMonitorProtocol): - def __init__(self, address, pp=None): + def __init__(self, address, pretty=False): qmp.QEMUMonitorProtocol.__init__(self, self.__get_address(address)) self._greeting = None self._completer = None - self._pp = pp + self._pretty = pretty self._transmode = False self._actions = list() @@ -196,16 +230,16 @@ class QMPShell(qmp.QEMUMonitorProtocol): return qmpcmd def _print(self, qmp): - jsobj = json.dumps(qmp) - if self._pp is not None: - self._pp.pprint(jsobj) - else: - print str(jsobj) + indent = None + if self._pretty: + indent = 4 + jsobj = json.dumps(qmp, indent=indent) + print str(jsobj) def _execute_cmd(self, cmdline): try: qmpcmd = self.__build_cmd(cmdline) - except Exception, e: + except Exception as e: print 'Error while parsing command line: %s' % e print 'command format: <command-name> ', print '[arg-name1=arg1] ... [arg-nameN=argN]' @@ -342,7 +376,7 @@ def main(): addr = '' qemu = None hmp = False - pp = None + pretty = False verbose = False try: @@ -352,9 +386,7 @@ def main(): fail_cmdline(arg) hmp = True elif arg == "-p": - if pp is not None: - fail_cmdline(arg) - pp = pprint.PrettyPrinter(indent=4) + pretty = True elif arg == "-v": verbose = True else: @@ -363,7 +395,7 @@ def main(): if hmp: qemu = HMPShell(arg) else: - qemu = QMPShell(arg, pp) + qemu = QMPShell(arg, pretty) addr = arg if qemu is None: diff --git a/qemu/scripts/qmp/qmp.py b/qemu/scripts/qmp/qmp.py index 1d38e3e9e..779332f32 100644 --- a/qemu/scripts/qmp/qmp.py +++ b/qemu/scripts/qmp/qmp.py @@ -92,7 +92,7 @@ class QEMUMonitorProtocol: self.__sock.setblocking(0) try: self.__json_read() - except socket.error, err: + except socket.error as err: if err[0] == errno.EAGAIN: # No data available pass @@ -150,7 +150,7 @@ class QEMUMonitorProtocol: """ try: self.__sock.sendall(json.dumps(qmp_cmd)) - except socket.error, err: + except socket.error as err: if err[0] == errno.EPIPE: return raise socket.error(err) diff --git a/qemu/scripts/texi2pod.pl b/qemu/scripts/texi2pod.pl index 94097fb06..8767662d3 100755 --- a/qemu/scripts/texi2pod.pl +++ b/qemu/scripts/texi2pod.pl @@ -317,7 +317,7 @@ while(<$inf>) { @columns = (); for $column (split (/\s*\@tab\s*/, $1)) { # @strong{...} is used a @headitem work-alike - $column =~ s/^\@strong{(.*)}$/$1/; + $column =~ s/^\@strong\{(.*)\}$/$1/; push @columns, $column; } $_ = "\n=item ".join (" : ", @columns)."\n"; diff --git a/qemu/scripts/tracetool.py b/qemu/scripts/tracetool.py index 83bde7bda..7b82959e8 100755 --- a/qemu/scripts/tracetool.py +++ b/qemu/scripts/tracetool.py @@ -71,7 +71,7 @@ def main(args): try: opts, args = getopt.getopt(args[1:], "", long_opts) - except getopt.GetoptError, err: + except getopt.GetoptError as err: error_opt(str(err)) check_backends = False @@ -132,7 +132,7 @@ def main(args): try: tracetool.generate(sys.stdin, arg_format, arg_backends, binary=binary, probe_prefix=probe_prefix) - except tracetool.TracetoolError, e: + except tracetool.TracetoolError as e: error_opt(str(e)) if __name__ == "__main__": diff --git a/qemu/scripts/tracetool/__init__.py b/qemu/scripts/tracetool/__init__.py index 181675f00..be24039c5 100644 --- a/qemu/scripts/tracetool/__init__.py +++ b/qemu/scripts/tracetool/__init__.py @@ -6,7 +6,7 @@ Machinery for generating tracing-related intermediate files. """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" @@ -50,9 +50,14 @@ class Arguments: Parameters ---------- args : - List of (type, name) tuples. + List of (type, name) tuples or Arguments objects. """ - self._args = args + self._args = [] + for arg in args: + if isinstance(arg, Arguments): + self._args.extend(arg._args) + else: + self._args.append(arg) def copy(self): """Create a new copy.""" @@ -83,6 +88,12 @@ class Arguments: res.append((arg_type, identifier)) return Arguments(res) + def __getitem__(self, index): + if isinstance(index, slice): + return Arguments(self._args[index]) + else: + return self._args[index] + def __iter__(self): """Iterate over the (type, name) pairs.""" return iter(self._args) @@ -110,6 +121,10 @@ class Arguments: """List of argument types.""" return [ type_ for type_, _ in self._args ] + def casted(self): + """List of argument names casted to their type.""" + return ["(%s)%s" % (type_, name) for type_, name in self._args] + def transform(self, *trans): """Return a new Arguments instance with transformed types. @@ -146,9 +161,10 @@ class Event(object): "(?:(?:(?P<fmt_trans>\".+),)?\s*(?P<fmt>\".+))?" "\s*") - _VALID_PROPS = set(["disable", "tcg", "tcg-trans", "tcg-exec"]) + _VALID_PROPS = set(["disable", "tcg", "tcg-trans", "tcg-exec", "vcpu"]) - def __init__(self, name, props, fmt, args, orig=None): + def __init__(self, name, props, fmt, args, orig=None, + event_trans=None, event_exec=None): """ Parameters ---------- @@ -161,13 +177,19 @@ class Event(object): args : Arguments Event arguments. orig : Event or None - Original Event before transformation. + Original Event before transformation/generation. + event_trans : Event or None + Generated translation-time event ("tcg" property). + event_exec : Event or None + Generated execution-time event ("tcg" property). """ self.name = name self.properties = props self.fmt = fmt self.args = args + self.event_trans = event_trans + self.event_exec = event_exec if orig is None: self.original = weakref.ref(self) @@ -183,7 +205,7 @@ class Event(object): def copy(self): """Create a new copy.""" return Event(self.name, list(self.properties), self.fmt, - self.args.copy(), self) + self.args.copy(), self, self.event_trans, self.event_exec) @staticmethod def build(line_str): @@ -215,7 +237,13 @@ class Event(object): if "tcg" in props and isinstance(fmt, str): raise ValueError("Events with 'tcg' property must have two formats") - return Event(name, props, fmt, args) + event = Event(name, props, fmt, args) + + # add implicit arguments when using the 'vcpu' property + import tracetool.vcpu + event = tracetool.vcpu.transform_event(event) + + return event def __repr__(self): """Evaluable string representation for this object.""" @@ -270,6 +298,7 @@ def _read_events(fobj): event_trans.name += "_trans" event_trans.properties += ["tcg-trans"] event_trans.fmt = event.fmt[0] + # ignore TCG arguments args_trans = [] for atrans, aorig in zip( event_trans.transform(tracetool.transform.TCG_2_HOST).args, @@ -277,13 +306,12 @@ def _read_events(fobj): if atrans == aorig: args_trans.append(atrans) event_trans.args = Arguments(args_trans) - event_trans = event_trans.copy() event_exec = event.copy() event_exec.name += "_exec" event_exec.properties += ["tcg-exec"] event_exec.fmt = event.fmt[1] - event_exec = event_exec.transform(tracetool.transform.TCG_2_HOST) + event_exec.args = event_exec.args.transform(tracetool.transform.TCG_2_HOST) new_event = [event_trans, event_exec] event.event_trans, event.event_exec = new_event diff --git a/qemu/scripts/tracetool/backend/stderr.py b/qemu/scripts/tracetool/backend/log.py index ca5805462..e409b7326 100644 --- a/qemu/scripts/tracetool/backend/stderr.py +++ b/qemu/scripts/tracetool/backend/log.py @@ -20,11 +20,8 @@ PUBLIC = True def generate_h_begin(events): - out('#include <stdio.h>', - '#include <sys/time.h>', - '#include <sys/types.h>', - '#include <unistd.h>', - '#include "trace/control.h"', + out('#include "trace/control.h"', + '#include "qemu/log.h"', '') @@ -36,10 +33,10 @@ def generate_h(event): out(' if (trace_event_get_state(%(event_id)s)) {', ' struct timeval _now;', ' gettimeofday(&_now, NULL);', - ' fprintf(stderr, "%%d@%%zd.%%06zd:%(name)s " %(fmt)s "\\n",', - ' getpid(),', - ' (size_t)_now.tv_sec, (size_t)_now.tv_usec', - ' %(argnames)s);', + ' qemu_log_mask(LOG_TRACE, "%%d@%%zd.%%06zd:%(name)s " %(fmt)s "\\n",', + ' getpid(),', + ' (size_t)_now.tv_sec, (size_t)_now.tv_usec', + ' %(argnames)s);', ' }', event_id="TRACE_" + event.name.upper(), name=event.name, diff --git a/qemu/scripts/tracetool/backend/simple.py b/qemu/scripts/tracetool/backend/simple.py index e8c2cd57e..3246c2001 100644 --- a/qemu/scripts/tracetool/backend/simple.py +++ b/qemu/scripts/tracetool/backend/simple.py @@ -42,7 +42,8 @@ def generate_h(event): def generate_c_begin(events): - out('#include "trace.h"', + out('#include "qemu/osdep.h"', + '#include "trace.h"', '#include "trace/control.h"', '#include "trace/simple.h"', '') diff --git a/qemu/scripts/tracetool/format/events_c.py b/qemu/scripts/tracetool/format/events_c.py index 2d97fa310..1cc6a49a7 100644 --- a/qemu/scripts/tracetool/format/events_c.py +++ b/qemu/scripts/tracetool/format/events_c.py @@ -19,6 +19,7 @@ from tracetool import out def generate(events, backend): out('/* This file is autogenerated by tracetool, do not edit. */', '', + '#include "qemu/osdep.h"', '#include "trace.h"', '#include "trace/generated-events.h"', '#include "trace/control.h"', @@ -27,7 +28,7 @@ def generate(events, backend): out('TraceEvent trace_events[TRACE_EVENT_COUNT] = {') for e in events: - out(' { .id = %(id)s, .name = \"%(name)s\", .sstate = %(sstate)s, .dstate = 0 },', + out(' { .id = %(id)s, .name = \"%(name)s\", .sstate = %(sstate)s },', id = "TRACE_" + e.name.upper(), name = e.name, sstate = "TRACE_%s_ENABLED" % e.name.upper()) diff --git a/qemu/scripts/tracetool/format/events_h.py b/qemu/scripts/tracetool/format/events_h.py index 9f114a349..4529263e0 100644 --- a/qemu/scripts/tracetool/format/events_h.py +++ b/qemu/scripts/tracetool/format/events_h.py @@ -6,7 +6,7 @@ trace/generated-events.h """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" @@ -21,8 +21,6 @@ def generate(events, backend): '', '#ifndef TRACE__GENERATED_EVENTS_H', '#define TRACE__GENERATED_EVENTS_H', - '', - '#include <stdbool.h>', '') # event identifiers @@ -43,7 +41,7 @@ def generate(events, backend): if "tcg-trans" in e.properties: # a single define for the two "sub-events" out('#define TRACE_%(name)s_ENABLED %(enabled)d', - name=e.original.original.name.upper(), + name=e.original.name.upper(), enabled=enabled) out('#define TRACE_%s_ENABLED %d' % (e.name.upper(), enabled)) diff --git a/qemu/scripts/tracetool/format/h.py b/qemu/scripts/tracetool/format/h.py index 9b3943002..083540621 100644 --- a/qemu/scripts/tracetool/format/h.py +++ b/qemu/scripts/tracetool/format/h.py @@ -6,7 +6,7 @@ trace/generated-tracers.h """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" diff --git a/qemu/scripts/tracetool/format/tcg_h.py b/qemu/scripts/tracetool/format/tcg_h.py index f676b6662..e2331f251 100644 --- a/qemu/scripts/tracetool/format/tcg_h.py +++ b/qemu/scripts/tracetool/format/tcg_h.py @@ -6,14 +6,25 @@ Generate .h file for TCG code generation. """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@linux.vnet.ibm.com" -from tracetool import out +from tracetool import out, Arguments +import tracetool.vcpu + + +def vcpu_transform_args(args): + assert len(args) == 1 + return Arguments([ + args, + # NOTE: this name must be kept in sync with the one in "tcg_h" + # NOTE: Current helper code uses TCGv_env (CPUArchState*) + ("TCGv_env", "__tcg_" + args.names()[0]), + ]) def generate(events, backend): @@ -23,8 +34,6 @@ def generate(events, backend): '#ifndef TRACE__GENERATED_TCG_TRACERS_H', '#define TRACE__GENERATED_TCG_TRACERS_H', '', - '#include <stdint.h>', - '', '#include "trace.h"', '#include "exec/helper-proto.h"', '', @@ -35,21 +44,21 @@ def generate(events, backend): if "tcg-trans" not in e.properties: continue - # get the original event definition - e = e.original.original - out('static inline void %(name_tcg)s(%(args)s)', '{', - name_tcg=e.api(e.QEMU_TRACE_TCG), - args=e.args) + name_tcg=e.original.api(e.QEMU_TRACE_TCG), + args=tracetool.vcpu.transform_args("tcg_h", e.original)) if "disable" not in e.properties: + args_trans = e.original.event_trans.args + args_exec = tracetool.vcpu.transform_args( + "tcg_helper_c", e.original.event_exec, "wrapper") out(' %(name_trans)s(%(argnames_trans)s);', ' gen_helper_%(name_exec)s(%(argnames_exec)s);', - name_trans=e.event_trans.api(e.QEMU_TRACE), - name_exec=e.event_exec.api(e.QEMU_TRACE), - argnames_trans=", ".join(e.event_trans.args.names()), - argnames_exec=", ".join(e.event_exec.args.names())) + name_trans=e.original.event_trans.api(e.QEMU_TRACE), + name_exec=e.original.event_exec.api(e.QEMU_TRACE), + argnames_trans=", ".join(args_trans.names()), + argnames_exec=", ".join(args_exec.names())) out('}') diff --git a/qemu/scripts/tracetool/format/tcg_helper_c.py b/qemu/scripts/tracetool/format/tcg_helper_c.py index 96655a059..a089b0bf0 100644 --- a/qemu/scripts/tracetool/format/tcg_helper_c.py +++ b/qemu/scripts/tracetool/format/tcg_helper_c.py @@ -6,15 +6,38 @@ Generate trace/generated-helpers.c. """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" __email__ = "stefanha@linux.vnet.ibm.com" -from tracetool import out +from tracetool import Arguments, out from tracetool.transform import * +import tracetool.vcpu + + +def vcpu_transform_args(args, mode): + assert len(args) == 1 + # NOTE: this name must be kept in sync with the one in "tcg_h" + args = Arguments([(args.types()[0], "__tcg_" + args.names()[0])]) + if mode == "code": + return Arguments([ + # Does cast from helper requirements to tracing types + ("CPUState *", "ENV_GET_CPU(%s)" % args.names()[0]), + ]) + else: + args = Arguments([ + # NOTE: Current helper code uses TCGv_env (CPUArchState*) + ("CPUArchState *", args.names()[0]), + ]) + if mode == "header": + return args + elif mode == "wrapper": + return args.transform(HOST_2_TCG) + else: + assert False def generate(events, backend): @@ -23,6 +46,7 @@ def generate(events, backend): out('/* This file is autogenerated by tracetool, do not edit. */', '', + '#include "qemu/osdep.h"', '#include "qemu-common.h"', '#include "trace.h"', '#include "exec/helper-proto.h"', @@ -33,18 +57,18 @@ def generate(events, backend): if "tcg-exec" not in e.properties: continue - # tracetool.generate always transforms types to host - e_args = e.original.args - - values = ["(%s)%s" % (t, n) - for t, n in e.args.transform(TCG_2_TCG_HELPER_DEF)] + e_args_api = tracetool.vcpu.transform_args( + "tcg_helper_c", e.original, "header").transform( + HOST_2_TCG_COMPAT, TCG_2_TCG_HELPER_DEF) + e_args_call = tracetool.vcpu.transform_args( + "tcg_helper_c", e, "code") - out('void %(name_tcg)s(%(args)s)', + out('void %(name_tcg)s(%(args_api)s)', '{', - ' %(name)s(%(values)s);', + ' %(name)s(%(args_call)s);', '}', name_tcg="helper_%s_proxy" % e.api(), name=e.api(), - args=e_args.transform(HOST_2_TCG_COMPAT, TCG_2_TCG_HELPER_DEF), - values=", ".join(values), + args_api=e_args_api, + args_call=", ".join(e_args_call.casted()), ) diff --git a/qemu/scripts/tracetool/format/tcg_helper_h.py b/qemu/scripts/tracetool/format/tcg_helper_h.py index a8ba7ba8e..dc76c15eb 100644 --- a/qemu/scripts/tracetool/format/tcg_helper_h.py +++ b/qemu/scripts/tracetool/format/tcg_helper_h.py @@ -6,7 +6,7 @@ Generate trace/generated-helpers.h. """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" @@ -15,6 +15,7 @@ __email__ = "stefanha@linux.vnet.ibm.com" from tracetool import out from tracetool.transform import * +import tracetool.vcpu def generate(events, backend): @@ -29,11 +30,9 @@ def generate(events, backend): if "tcg-exec" not in e.properties: continue - # tracetool.generate always transforms types to host - e_args = e.original.args - # TCG helper proxy declaration fmt = "DEF_HELPER_FLAGS_%(argc)d(%(name)s, %(flags)svoid%(types)s)" + e_args = tracetool.vcpu.transform_args("tcg_helper_c", e.original, "header") args = e_args.transform(HOST_2_TCG_COMPAT, HOST_2_TCG, TCG_2_TCG_HELPER_DECL) types = ", ".join(args.types()) diff --git a/qemu/scripts/tracetool/format/tcg_helper_wrapper_h.py b/qemu/scripts/tracetool/format/tcg_helper_wrapper_h.py index cac5a878f..020f4422a 100644 --- a/qemu/scripts/tracetool/format/tcg_helper_wrapper_h.py +++ b/qemu/scripts/tracetool/format/tcg_helper_wrapper_h.py @@ -6,7 +6,7 @@ Generate trace/generated-helpers-wrappers.h. """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" @@ -15,6 +15,7 @@ __email__ = "stefanha@linux.vnet.ibm.com" from tracetool import out from tracetool.transform import * +import tracetool.vcpu def generate(events, backend): @@ -33,7 +34,7 @@ def generate(events, backend): continue # tracetool.generate always transforms types to host - e_args = e.original.args + e_args = tracetool.vcpu.transform_args("tcg_helper_c", e.original, "wrapper") # mixed-type to TCG helper bridge args_tcg_compat = e_args.transform(HOST_2_TCG_COMPAT) diff --git a/qemu/scripts/tracetool/format/ust_events_c.py b/qemu/scripts/tracetool/format/ust_events_c.py index bc970936b..9967c7a82 100644 --- a/qemu/scripts/tracetool/format/ust_events_c.py +++ b/qemu/scripts/tracetool/format/ust_events_c.py @@ -22,6 +22,8 @@ def generate(events, backend): out('/* This file is autogenerated by tracetool, do not edit. */', '', + '#include "qemu/osdep.h"', + '', '#define TRACEPOINT_DEFINE', '#define TRACEPOINT_CREATE_PROBES', '', diff --git a/qemu/scripts/tracetool/transform.py b/qemu/scripts/tracetool/transform.py index fc5e679ed..e18b05315 100644 --- a/qemu/scripts/tracetool/transform.py +++ b/qemu/scripts/tracetool/transform.py @@ -6,7 +6,7 @@ Type-transformation rules. """ __author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" -__copyright__ = "Copyright 2012-2014, Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2012-2016, Lluís Vilanova <vilanova@ac.upc.edu>" __license__ = "GPL version 2 or (at your option) any later version" __maintainer__ = "Stefan Hajnoczi" @@ -98,6 +98,7 @@ HOST_2_TCG = { "uint32_t": "TCGv_i32", "uint64_t": "TCGv_i64", "void *" : "TCGv_ptr", + "CPUArchState *": "TCGv_env", None: _host_2_tcg, } @@ -130,6 +131,7 @@ TCG_2_TCG_HELPER_DECL = { "TCGv_ptr": "ptr", "TCGv_i32": "i32", "TCGv_i64": "i64", + "TCGv_env": "env", None: _tcg_2_tcg_helper_decl_error, } diff --git a/qemu/scripts/tracetool/vcpu.py b/qemu/scripts/tracetool/vcpu.py new file mode 100644 index 000000000..452c7f589 --- /dev/null +++ b/qemu/scripts/tracetool/vcpu.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +Generic management for the 'vcpu' property. + +""" + +__author__ = "Lluís Vilanova <vilanova@ac.upc.edu>" +__copyright__ = "Copyright 2016, Lluís Vilanova <vilanova@ac.upc.edu>" +__license__ = "GPL version 2 or (at your option) any later version" + +__maintainer__ = "Stefan Hajnoczi" +__email__ = "stefanha@linux.vnet.ibm.com" + + +from tracetool import Arguments, try_import + + +def transform_event(event): + """Transform event to comply with the 'vcpu' property (if present).""" + if "vcpu" in event.properties: + # events with 'tcg-trans' and 'tcg-exec' are auto-generated from + # already-patched events + assert "tcg-trans" not in event.properties + assert "tcg-exec" not in event.properties + + event.args = Arguments([("CPUState *", "__cpu"), event.args]) + if "tcg" in event.properties: + fmt = "\"cpu=%p \"" + event.fmt = [fmt + event.fmt[0], + fmt + event.fmt[1]] + else: + fmt = "\"cpu=%p \"" + event.fmt = fmt + event.fmt + return event + + +def transform_args(format, event, *args, **kwargs): + """Transforms the arguments to suit the specified format. + + The format module must implement function 'vcpu_args', which receives the + implicit arguments added by the 'vcpu' property, and must return suitable + arguments for the given format. + + The function is only called for events with the 'vcpu' property. + + Parameters + ========== + format : str + Format module name. + event : Event + args, kwargs + Passed to 'vcpu_transform_args'. + + Returns + ======= + Arguments + The transformed arguments, including the non-implicit ones. + + """ + if "vcpu" in event.properties: + ok, func = try_import("tracetool.format." + format, + "vcpu_transform_args") + assert ok + assert func + return Arguments([func(event.args[:1], *args, **kwargs), + event.args[1:]]) + else: + return event.args diff --git a/qemu/scripts/update-acpi.sh b/qemu/scripts/update-acpi.sh deleted file mode 100644 index b5f05ff3c..000000000 --- a/qemu/scripts/update-acpi.sh +++ /dev/null @@ -1,4 +0,0 @@ -cd x86_64-softmmu -for file in hw/i386/*.hex; do - cp -f $file ../$file.generated -done diff --git a/qemu/scripts/update-linux-headers.sh b/qemu/scripts/update-linux-headers.sh index f0e830c2d..f7d62d974 100755 --- a/qemu/scripts/update-linux-headers.sh +++ b/qemu/scripts/update-linux-headers.sh @@ -28,38 +28,35 @@ if [ -z "$output" ]; then output="$PWD" fi -cp_virtio() { - from=$1 +cp_portable() { + f=$1 to=$2 - virtio=$(find "$from" -name '*virtio*h' -o -name "input.h" -o -name "pci_regs.h") - if [ "$virtio" ]; then - rm -rf "$to" - mkdir -p "$to" - for f in $virtio; do - if - grep '#include' "$f" | grep -v -e 'linux/virtio' \ - -e 'linux/types' \ - -e 'linux/if_ether' \ - -e 'sys/' \ - > /dev/null - then - echo "Unexpected #include in input file $f". - exit 2 - fi - - header=$(basename "$f"); - sed -e 's/__u\([0-9][0-9]*\)/uint\1_t/g' \ - -e 's/__s\([0-9][0-9]*\)/int\1_t/g' \ - -e 's/__le\([0-9][0-9]*\)/uint\1_t/g' \ - -e 's/__be\([0-9][0-9]*\)/uint\1_t/g' \ - -e 's/<linux\/\([^>]*\)>/"standard-headers\/linux\/\1"/' \ - -e 's/__bitwise__//' \ - -e 's/__attribute__((packed))/QEMU_PACKED/' \ - -e 's/__inline__/inline/' \ - -e '/sys\/ioctl.h/d' \ - "$f" > "$to/$header"; - done + if + grep '#include' "$f" | grep -v -e 'linux/virtio' \ + -e 'linux/types' \ + -e 'stdint' \ + -e 'linux/if_ether' \ + -e 'input-event-codes' \ + -e 'sys/' \ + > /dev/null + then + echo "Unexpected #include in input file $f". + exit 2 fi + + header=$(basename "$f"); + sed -e 's/__u\([0-9][0-9]*\)/uint\1_t/g' \ + -e 's/__s\([0-9][0-9]*\)/int\1_t/g' \ + -e 's/__le\([0-9][0-9]*\)/uint\1_t/g' \ + -e 's/__be\([0-9][0-9]*\)/uint\1_t/g' \ + -e 's/"\(input-event-codes\.h\)"/"standard-headers\/linux\/\1"/' \ + -e 's/<linux\/\([^>]*\)>/"standard-headers\/linux\/\1"/' \ + -e 's/__bitwise__//' \ + -e 's/__attribute__((packed))/QEMU_PACKED/' \ + -e 's/__inline__/inline/' \ + -e '/sys\/ioctl.h/d' \ + -e 's/SW_MAX/SW_MAX_/' \ + "$f" > "$to/$header"; } # This will pick up non-directories too (eg "Kconfig") but we will @@ -74,7 +71,7 @@ for arch in $ARCHLIST; do fi # Blacklist architectures which have KVM headers but are actually dead - if [ "$arch" = "ia64" ]; then + if [ "$arch" = "ia64" -o "$arch" = "mips" ]; then continue fi @@ -82,23 +79,31 @@ for arch in $ARCHLIST; do rm -rf "$output/linux-headers/asm-$arch" mkdir -p "$output/linux-headers/asm-$arch" - for header in kvm.h kvm_para.h; do + for header in kvm.h kvm_para.h unistd.h; do cp "$tmpdir/include/asm/$header" "$output/linux-headers/asm-$arch" done - if [ $arch = x86 ]; then - cp "$tmpdir/include/asm/hyperv.h" "$output/linux-headers/asm-x86" - fi if [ $arch = powerpc ]; then cp "$tmpdir/include/asm/epapr_hcalls.h" "$output/linux-headers/asm-powerpc/" fi - cp_virtio "$tmpdir/include/asm" "$output/include/standard-headers/asm-$arch" + rm -rf "$output/include/standard-headers/asm-$arch" + mkdir -p "$output/include/standard-headers/asm-$arch" + if [ $arch = s390 ]; then + cp_portable "$tmpdir/include/asm/kvm_virtio.h" "$output/include/standard-headers/asm-s390/" + cp_portable "$tmpdir/include/asm/virtio-ccw.h" "$output/include/standard-headers/asm-s390/" + fi + if [ $arch = x86 ]; then + cp_portable "$tmpdir/include/asm/hyperv.h" "$output/include/standard-headers/asm-x86/" + cp "$tmpdir/include/asm/unistd_32.h" "$output/linux-headers/asm-x86/" + cp "$tmpdir/include/asm/unistd_x32.h" "$output/linux-headers/asm-x86/" + cp "$tmpdir/include/asm/unistd_64.h" "$output/linux-headers/asm-x86/" + fi done rm -rf "$output/linux-headers/linux" mkdir -p "$output/linux-headers/linux" for header in kvm.h kvm_para.h vfio.h vhost.h \ - psci.h; do + psci.h userfaultfd.h; do cp "$tmpdir/include/linux/$header" "$output/linux-headers/linux" done rm -rf "$output/linux-headers/asm-generic" @@ -112,6 +117,9 @@ else cp "$linux/COPYING" "$output/linux-headers" fi +cat <<EOF >$output/linux-headers/asm-x86/hyperv.h +#include "standard-headers/asm-x86/hyperv.h" +EOF cat <<EOF >$output/linux-headers/linux/virtio_config.h #include "standard-headers/linux/virtio_config.h" EOF @@ -119,11 +127,18 @@ cat <<EOF >$output/linux-headers/linux/virtio_ring.h #include "standard-headers/linux/virtio_ring.h" EOF -cp_virtio "$tmpdir/include/linux/" "$output/include/standard-headers/linux" +rm -rf "$output/include/standard-headers/linux" +mkdir -p "$output/include/standard-headers/linux" +for i in "$tmpdir"/include/linux/*virtio*.h "$tmpdir/include/linux/input.h" \ + "$tmpdir/include/linux/input-event-codes.h" \ + "$tmpdir/include/linux/pci_regs.h"; do + cp_portable "$i" "$output/include/standard-headers/linux" +done cat <<EOF >$output/include/standard-headers/linux/types.h -#include <stdint.h> -#include "qemu/compiler.h" +/* For QEMU all types are already defined via osdep.h, so this + * header does not need to do anything. + */ EOF cat <<EOF >$output/include/standard-headers/linux/if_ether.h #define ETH_ALEN 6 diff --git a/qemu/scripts/vmstate-static-checker.py b/qemu/scripts/vmstate-static-checker.py index b6c0bbead..b5ecaf644 100755 --- a/qemu/scripts/vmstate-static-checker.py +++ b/qemu/scripts/vmstate-static-checker.py @@ -99,6 +99,7 @@ def get_changed_sec_name(sec): # Section names can change -- see commit 292b1634 for an example. changes = { "ICH9 LPC": "ICH9-LPC", + "e1000-82540em": "e1000", } for item in changes: |