10.7. Regex Syntax Group

• Catch expression results

• Can be named or positional

• Note, that for backreference, must use raw-sting or double backslash

• () - matches whatever regular expression is inside the parentheses, and indicates the start and end of a group

• (...) - unnamed group

• (?P<mygroup>...) - named group mygroup

• (?:...) - non-capturing group

• (?#...) - comment

10.7.1. SetUp

>>> import re

10.7.2. Positional Group

• (...) - unnamed (positional) group

>>> TEXT = 'Email from Mark Watney <mwatney@nasa.gov> received on: Sat, Jan 1st, 2000 at 12:00 AM'

>>> re.findall(r'\dst', TEXT)
['1st']
>>>
>>> re.findall(r'(\d{1,2})st', TEXT)
['1']
>>>
>>> re.findall(r'\d{1,2}(st)', TEXT)
['st']
>>> re.findall(r'(\d{1,2})(st)', TEXT)
[('1', 'st')]

>>> re.findall(r'\d{1,2}:\d{2}', TEXT)
['12:00']
>>>
>>> re.findall(r'(\d{1,2}):\d{2}', TEXT)
['12']
>>>
>>> re.findall(r'\d{1,2}:(\d{2})', TEXT)
['00']
>>>
>>> re.findall(r'(\d{1,2}):(\d{2})', TEXT)
[('12', '00')]

>>> re.findall(r'([A-Z][a-z]+\s[A-Z][a-z]+)', TEXT)
['Mark Watney']
>>>
>>> re.findall(r'([A-Z][a-z]+) ([A-Z][a-z]+)', TEXT)
[('Mark', 'Watney')]
>>>
>>> re.findall(r'([A-Z][a-z]+) ([A-Z][a-z]+)', TEXT)[0]
('Mark', 'Watney')

>>> firstname = r'([A-Z][a-z]+)'
>>> lastname = r'([A-Z][a-z]+)'
>>>
>>> re.findall(f'{firstname} {lastname}', TEXT)[0]
('Mark', 'Watney')

>>> firstname = r'([A-Z][a-z]+)'
>>> lastname = r'([A-Z][a-z]+)'
>>> name = f'{firstname} {lastname}'
>>>
>>> re.findall(name, TEXT)[0]
('Mark', 'Watney')

>>> firstname = r'[A-Z][a-z]+'
>>> lastname = r'[A-Z][a-z]+'
>>> name = f'({firstname}) ({lastname})'
>>>
>>> re.findall(name, TEXT)[0]
('Mark', 'Watney')

10.7.3. Named Group

• (?P<mygroup>...) - named group mygroup

>>> import re
>>> TEXT = 'Email from Mark Watney <mwatney@nasa.gov> received on: Sat, Jan 1st, 2000 at 12:00 AM'

>>> firstname = r'[A-Z][a-z]+'
>>> lastname = r'[A-Z][a-z]+'
>>> name = f'(?P<firstname>{firstname}) (?P<lastname>{lastname})'
>>>
>>> re.findall(name, TEXT)
[('Mark', 'Watney')]
>>>
>>> re.search(name, TEXT)
<re.Match object; span=(11, 22), match='Mark Watney'>
>>>
>>> re.search(name, TEXT).groups()
('Mark', 'Watney')
>>>
>>> re.search(name, TEXT).groupdict()
{'firstname': 'Mark', 'lastname': 'Watney'}

>>> TEXT = 'Email from Mark Watney <mwatney@nasa.gov> received on: Sat, Jan 1st, 2000 at 12:00 AM'
>>> time = r'(?P<hour>\d{1,2}):(?P<minute>\d{1,2})'
>>>
>>> re.findall(time, TEXT)
[('12', '00')]
>>>
>>> re.search(time, TEXT).groups()
('12', '00')
>>>
>>> re.search(time, TEXT).group(0)
'12:00'
>>>
>>> re.search(time, TEXT).group(1)
'12'
>>>
>>> re.search(time, TEXT).group(2)
'00'
>>>
>>> re.search(time, TEXT).groupdict()
{'hour': '12', 'minute': '00'}

10.7.4. Non-Capturing Group

• (?:...)

>>> import re
>>> TEXT = 'Email from Mark Watney <mwatney@nasa.gov> received on: Sat, Jan 1st, 2000 at 12:00 AM'

>>> re.findall(r'\w{3} \d{1,2}st, \d{4}', TEXT)
['Jan 1st, 2000']
>>>
>>> re.findall(r'\w{3} \d{1,2}st|nd|rd|th, \d{4}', TEXT)
['Jan 1st']
>>>
>>> re.findall(r'\w{3} \d{1,2}(st|nd|rd|th), \d{4}', TEXT)
['st']
>>>
>>> re.findall(r'\w{3} \d{1,2}(?:st|nd|rd|th), \d{4}', TEXT)
['Jan 1st, 2000']
>>>
>>> re.findall(r'(\w{3}) (\d{1,2})(?:st|nd|rd|th), (\d{4})', TEXT)
[('Jan', '1', '2000')]
>>>
>>> re.findall(r'(\w{3}) (\d{1,2})(st|nd|rd|th), (\d{4})', TEXT)
[('Jan', '1', 'st', '2000')]

>>> date = r'(\w{3} \d{1,2}(?:st|nd|rd|th), \d{4})'
>>> re.findall(date, TEXT)
['Jan 1st, 2000']

>>> year = r'\d{4}'
>>> month = r'\w{3}'
>>> day = r'\d{1,2}'
>>>
>>> re.findall(f'{month} {day}(st|nd|rd|th), {year}', TEXT)
['st']
>>>
>>> re.findall(f'{month} {day}(?:st|nd|rd|th), {year}', TEXT)
['Jan 1st, 2000']

10.7.5. Comment

• (?#...) - comment

>>> import re
>>> TEXT = 'Email from Mark Watney <mwatney@nasa.gov> received on: Sat, Jan 1st, 2000 at 12:00 AM'

>>> re.findall(r'\d{4}(?#year)', TEXT)
['2000']
>>>
>>> re.findall(r'\d{1,2}(?#hour):\d{2}(?#minute)', TEXT)
['12:00']

>>> hour = r'\d{1,2}(?#hour)'
>>> minute = r'\d{2}(?#minute)'
>>> time = f'{hour}:{minute}'
>>>
>>> re.findall(time, TEXT)
['12:00']
>>>
>>> time
'\\d{1,2}(?#hour):\\d{2}(?#minute)'

10.7.6. Backreference

• \g<number> - backreferencing by group number

• \g<name> - backreferencing by group name

• (?P=name) - backreferencing by group name

• \number - backreferencing by group number

>>> year = r'(?P<year>\d{4})'
>>> month = r'(?P<month>[A-Z][a-z]{2})'
>>> day = r'(?P<day>\d{1,2})'
>>> date = f'{month} {day}(?:st|nd|rd|th), {year}'
>>>
>>> TEXT = 'Email from Mark Watney <mwatney@nasa.gov> received on: Sat, Jan 1st, 2000 at 12:00 AM'
>>>
>>>
>>> re.sub(date, '\g<3> \g<1> \g<2>', TEXT)
'Email from Mark Watney <mwatney@nasa.gov> received on: Sat, 2000 Jan 1 at 12:00 AM'
>>>
>>> re.sub(date, '\g<year> \g<month> \g<day>', TEXT)
'Email from Mark Watney <mwatney@nasa.gov> received on: Sat, 2000 Jan 1 at 12:00 AM'

Although this is not working in Python:

>>> re.sub(f'{month} {day}st, {year}', '(?P=day) (?P=month) (?P=year)', TEXT)
'Email from Mark Watney <mwatney@nasa.gov> received on: Sat, (?P=day) (?P=month) (?P=year) at 12:00 AM'

Example:

>>> html = '<p>We choose to go to the <strong>Moon</strong></p>'
>>>
>>>
>>> re.findall('<(?P<tagname>[a-z]+)>.*</(?P=tagname)>', html)
['p']
>>>
>>> re.findall('<(?P<tagname>[a-z]+)>(.*)</(?P=tagname)>', html)
[('p', 'We choose to go to the <strong>Moon</strong>')]

10.7.7. Examples

• (\w+) - word character (including unicode chars, numbers an underscores)

• \d+(\.\d+)? - float with optional decimals

• \d+(,\d+)? - number with coma (,) as thousands separator

• (?P<word>\w+) - name group word with \w+ with at least one word character (including unicode chars, numbers an underscores)

• (?P<tag><.*?>).+(?P=tag) - matches text inside of a <tag> (opening and closing tag is the same)

• (.+) \1 - matches the the or 55 55

• (.+) \1 - not matches thethe (note the space after the group)

>>> import re
>>> TEXT = 'Email from Mark Watney <mwatney@nasa.gov> received on: Sat, Jan 1st, 2000 at 12:00 AM'

>>> re.findall(r'\d{,2}(st|nd|rd|th)?', TEXT)  
['', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '',
 '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '',
 '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '', '',
 '', '', '', '', '', '', '', 'st', '', '', '', '', '', '', '', '', '', '', '',
 '', '', '', '']
>>>
>>> re.findall(r'\d{1,2}(st|nd|rd|th)?', TEXT)
['st', '', '', '', '']
>>>
>>> re.findall(r'\d{1,2}(st|nd|rd|th)+?', TEXT)
['st']
>>>
>>> re.findall(r'\d{1,2}st|nd|rd|th+?', TEXT)
['1st']
>>>
>>> re.findall(r'\d{1,2}(?:st|nd|rd|th)+?', TEXT)
['1st']
>>>
>>> re.findall(r'(\d{1,2})(st|nd|rd|th)+?', TEXT)
[('1', 'st')]
>>>
>>> re.findall(r'(\d{1,2})(?:st|nd|rd|th)+?', TEXT)
['1']
>>>
>>> re.findall(r'(\w{3}) (\d{1,2})(?:st|nd|rd|th)+?, (\d{4})', TEXT)
[('Jan', '1', '2000')]
>>>
>>> re.findall(r'(\w{3}) (\d{1,2})(?:st|nd|rd|th)+?, (\d{4})', TEXT)[0]
('Jan', '1', '2000')
>>>
>>> re.findall(r'(\w{3} \d{1,2}(?:st|nd|rd|th)+?, \d{4})', TEXT)
['Jan 1st, 2000']

10.7.8. Use Case - 0x01

• Dates

>>> import re
>>> TEXT = 'Email from Mark Watney <mwatney@nasa.gov> received on: Sat, Jan 1st, 2000 at 12:00 AM'

>>> year = r'(?P<year>\d{4})'
>>> month = r'(?P<month>\w{3})'
>>> day = r'(?P<day>\d{1,2}(?:st|nd|rd|th)+?)'
>>> date = f'{month} {day}, {year}'
>>>
>>> re.search(date, TEXT).groupdict()
{'month': 'Jan', 'day': '1st', 'year': '2000'}

10.7.9. Use Case - 0x02

>>> import re

>>> line = 'value=123'
>>>
>>> re.findall(r'(\w+)\s?=\s?(\d+)', line)
[('value', '123')]

>>> line = 'value = 123'
>>>
>>> re.findall(r'(\w+)\s?=\s?(\d+)', line)
[('value', '123')]

10.7.10. Use Case - 0x03

>>> import re
>>>
>>>
>>> variable = r'(?P<variable>\w+)'
>>> space = r'\s?'  # optional space
>>> value = r'(?P<value>.+)'
>>> assignment = f'^{variable}{space}={space}{value}$'
>>>
>>> line_of_code = 'myvar = 123'
>>> re.findall(assignment, line_of_code)
[('myvar', '123')]

10.7.11. Use Case - 0x04

>>> import re
>>>
>>>
>>> variable = r'(?P<variable>\w+)'
>>> space = r'\s?(?#optional space)'
>>> value = r'(?P<value>.+)'
>>> assignment = f'^{variable}{space}={space}{value}$'
>>>
>>> assignment
'^(?P<variable>\\w+)\\s?(?#optional space)=\\s?(?#optional space)(?P<value>.+)$'

10.7.12. Use Case - 0x05

>>> import re
>>>
>>>
>>> HTML = '<p>Hello World</p>'
>>>
>>> search = r'<p>(.+)</p>'
>>> replace = r'<strong>\g<1></strong>'
>>>
>>> re.sub(search, replace, HTML)
'<strong>Hello World</strong>'

10.7.13. Use Case - 0x06

>>> import re
>>>
>>>
>>> HTML = '<p>Hello World</p>'
>>>
>>> search = r'<p>(?P<text>.+)</p>'
>>> replace = r'<strong>\g<text></strong>'
>>>
>>> re.sub(search, replace, HTML)
'<strong>Hello World</strong>'

10.7.14. Use Case - 0x07

>>> import re
>>>
>>>
>>> HTML = '<p>Hello World</p>'
>>> tag = re.findall(r'<(?P<tag>.+)>(?:.+)</(?P=tag)>', HTML)
>>>
>>> tag
['p']

10.7.15. Use Case - 0x08

>>> import re
>>>
>>>
>>> HTML = '<p>Hello World</p>'
>>>
>>> re.findall(r'<(?P<tag>.*?)>(.*?)</(?P=tag)>', HTML)
[('p', 'Hello World')]

10.7.16. Assignments

Code 10.37. Solution

"""
* Assignment: RE Syntax Group
* Complexity: easy
* Lines of code: 1 lines
* Time: 3 min

English:
    1. Define `result: str` with regular expression to find:
        a. all dates (month name followed by day number)
    2. Run doctests - all must succeed

Polish:
    1. Zdefiniuj `result: str` z wyrażeniem regularnym aby wyszukać:
        a. wszyskie daty (miesiąc po którym jest dzień)
    2. Uruchom doctesty - wszystkie muszą się powieść

Hint:
    * After day there is an ordinal: st, nd, rd, th (you can use: ..)

References:
    [1] Authors: Wikipedia contributors
        Title: Apollo 11
        Publisher: Wikipedia
        Year: 2019
        Retrieved: 2019-12-14
        URL: https://en.wikipedia.org/wiki/Apollo_11

Tests:
    >>> import sys; sys.tracebacklimit = 0
    >>> from pprint import pprint

    >>> result = re.findall(result, TEXT)
    >>> pprint(result, compact=True)
    ['July 20', 'July 21']
"""

import re

TEXT = """Apollo 11 was the American spaceflight that first landed
humans on the Moon. Commander (CDR) Neil Armstrong and lunar module
pilot (LMP) Buzz Aldrin landed the Apollo Lunar Module (LM) Eagle on
July 20th, 1969 at 20:17 UTC, and Armstrong became the first person
to step (EVA) onto the Moon's surface (EVA) 6 hours 39 minutes later,
on July 21st, 1969 at 02:56:15 UTC. Aldrin joined him 19 minutes later.
They spent 2 hours 31 minutes exploring the site they had named Tranquility
Base upon landing. Armstrong and Aldrin collected 47.5 pounds (21.5 kg)
of lunar material to bring back to Earth as pilot Michael Collins (CMP)
flew the Command Module (CM) Columbia in lunar orbit, and were on the
Moon's surface for 21 hours 36 minutes before lifting off to rejoin
Columbia."""

# Find all dates (month name followed by day number)
# Note: after day there is an ordinal: st, nd, rd, th (you can use: ..)
# Example: 'July 20', 'July 21'
# Note: define only regex pattern (str), not re.findall(...)
# type: str
result = ...

Code 10.38. Solution

"""
* Assignment: RE Syntax Group
* Complexity: easy
* Lines of code: 2 lines
* Time: 5 min

English:
    1. Define `result: str` with regular expression to find:
        a. all duration values, SKIP durations without hours
        b. all duration values, DO NOT SKIP durations without hours
    2. Use positional group
    3. Run doctests - all must succeed

Polish:
    1. Zdefiniuj `result: str` z wyrażeniem regularnym aby wyszukać:
       a.
    2. Użyj grup pozycyjnych
    3. Uruchom doctesty - wszystkie muszą się powieść

References:
    [1] Authors: Wikipedia contributors
        Title: Apollo 11
        Publisher: Wikipedia
        Year: 2019
        Retrieved: 2019-12-14
        URL: https://en.wikipedia.org/wiki/Apollo_11

Tests:
    >>> import sys; sys.tracebacklimit = 0
    >>> from pprint import pprint

    >>> result_a = re.findall(result_a, TEXT)
    >>> pprint(result_a, compact=True, width=20)
    [('6', '39'),
     ('2', '31'),
     ('21', '36')]

    >>> result_b = re.findall(result_b, TEXT)
    >>> pprint(result_b, compact=True, width=20)
    [('6', '39'),
     ('', '19'),
     ('2', '31'),
     ('21', '36')]
"""

import re

TEXT = """Apollo 11 was the American spaceflight that first landed
humans on the Moon. Commander (CDR) Neil Armstrong and lunar module
pilot (LMP) Buzz Aldrin landed the Apollo Lunar Module (LM) Eagle on
July 20th, 1969 at 20:17 UTC, and Armstrong became the first person
to step (EVA) onto the Moon's surface (EVA) 6 hours 39 minutes later,
on July 21st, 1969 at 02:56:15 UTC. Aldrin joined him 19 minutes later.
They spent 2 hours 31 minutes exploring the site they had named Tranquility
Base upon landing. Armstrong and Aldrin collected 47.5 pounds (21.5 kg)
of lunar material to bring back to Earth as pilot Michael Collins (CMP)
flew the Command Module (CM) Columbia in lunar orbit, and were on the
Moon's surface for 21 hours 36 minutes before lifting off to rejoin
Columbia."""

# Find all duration values, use positional groups
# SKIP durations without hours: 19 minutes later
# Example: [('6', '39'), ('2', '31'), ('21', '36')]
# Note: define only regex pattern (str), not re.findall(...)
# type: str
result_a = ...

# Find all duration values, use positional groups
# DO NOT SKIP durations without hours: 19 minutes later
# Example: [('6', '39'), ('', '19'), ('2', '31'), ('21', '36')]
# Note: define only regex pattern (str), not re.findall(...)
# type: str
result_b = ...

Code 10.39. Solution

"""
* Assignment: RE Syntax Group
* Complexity: easy
* Lines of code: 2 lines
* Time: 5 min

English:
    1. Define `result: str` with regular expression to find:
        a. all duration values, SKIP durations without hours
        b. all duration values, DO NOT SKIP durations without hours
    2. Use named group
    3. Run doctests - all must succeed

Polish:
    1. Zdefiniuj `result: str` z wyrażeniem regularnym aby wyszukać:
       a. wszystkie okresy czasowe, POMIŃ okresy bez godzin
       b. wszystkie okresy czasowe, NIE POMIJAJ okresów bez godzin
    2. Użyj grup nazwanych
    3. Uruchom doctesty - wszystkie muszą się powieść

References:
    [1] Authors: Wikipedia contributors
        Title: Apollo 11
        Publisher: Wikipedia
        Year: 2019
        Retrieved: 2019-12-14
        URL: https://en.wikipedia.org/wiki/Apollo_11

Tests:
    >>> import sys; sys.tracebacklimit = 0
    >>> from pprint import pprint

    >>> result_a = re.finditer(result_a, TEXT)
    >>> result_a = [x.groupdict() for x in result_a]
    >>> pprint(result_a, compact=True, width=50)
    [{'hours': '6', 'minutes': '39'},
     {'hours': '2', 'minutes': '31'},
     {'hours': '21', 'minutes': '36'}]

    >>> result_b = re.finditer(result_b, TEXT)
    >>> result_b = [x.groupdict() for x in result_b]
    >>> pprint(result_b, compact=True, width=50)
    [{'hours': '6', 'minutes': '39'},
     {'hours': None, 'minutes': '19'},
     {'hours': '2', 'minutes': '31'},
     {'hours': '21', 'minutes': '36'}]
"""

import re

TEXT = """Apollo 11 was the American spaceflight that first landed
humans on the Moon. Commander (CDR) Neil Armstrong and lunar module
pilot (LMP) Buzz Aldrin landed the Apollo Lunar Module (LM) Eagle on
July 20th, 1969 at 20:17 UTC, and Armstrong became the first person
to step (EVA) onto the Moon's surface (EVA) 6 hours 39 minutes later,
on July 21st, 1969 at 02:56:15 UTC. Aldrin joined him 19 minutes later.
They spent 2 hours 31 minutes exploring the site they had named Tranquility
Base upon landing. Armstrong and Aldrin collected 47.5 pounds (21.5 kg)
of lunar material to bring back to Earth as pilot Michael Collins (CMP)
flew the Command Module (CM) Columbia in lunar orbit, and were on the
Moon's surface for 21 hours 36 minutes before lifting off to rejoin
Columbia."""

# Find all duration values, use named groups
# SKIP durations without hours: 19 minutes later
# Example: [{'hours': '6', 'minutes': '39'}, {'hours': '2', 'minutes': '31'}]
# Note: define only regex pattern (str), not re.findall(...)
# type: str
result_a = ...

# Find all duration values, use named groups
# DO NOT SKIP durations without hours: 19 minutes later
# Example: [{'hours': '6', 'minutes': '39'}, {'hours': None, 'minutes': '19'}]
# Note: define only regex pattern (str), not re.findall(...)
# type: str
result_b = ...