Itertools & Functools #
These two modules are key to writing more expressive and efficient Python code. itertools provides building blocks for iteration — processing data lazily without loading everything into memory at once. functools provides tools for functional programming — modifying, combining, and optimizing functions. They’re often used together, and understanding them is a mark of mature Python code.
The itertools Module
#
The itertools module provides three main categories of iterator functions specifically designed to process data streams efficiently and memory-frugally. The classification of these three iterator types can be seen in the following diagram:
flowchart TD
Itertools["itertools Iterator Categories"] --> Inf["Infinite Iterators (Unbounded Iteration)"]
Itertools --> Term["Terminating Iterators (Bounded Iteration)"]
Itertools --> Comb["Combinatoric Iterators (Combinatorics)"]
Inf --> Inf1["count()"]
Inf --> Inf2["cycle()"]
Inf --> Inf3["repeat()"]
Term --> Term1["chain()"]
Term --> Term2["islice()"]
Term --> Term3["groupby()"]
Term --> TermOthers["Others (accumulate, compress, etc.)"]
Comb --> Comb1["product()"]
Comb --> Comb2["permutations()"]
Comb --> Comb3["combinations()"]chain — Combining Several Iterables
#
chain combines several iterables as if they were one, without making copies in memory.
from itertools import chain
# ANTI-PATTERN: combining with + (creates a new list in memory)
hasil = [1, 2, 3] + [4, 5, 6] + [7, 8, 9]
# CORRECT: chain doesn't make copies, processes one at a time
for item in chain([1, 2, 3], [4, 5, 6], [7, 8, 9]):
print(item)
# chain.from_iterable() -- for lists of lists
data = [[1, 2], [3, 4], [5, 6]]
# ANTI-PATTERN: flattening with a nested list comprehension
flat = [x for sublist in data for x in sublist]
# CORRECT: chain.from_iterable()
flat = list(chain.from_iterable(data))
print(flat) # [1, 2, 3, 4, 5, 6]
# Real example: combining query results from several tables
def ambil_semua_produk():
produk_elektronik = ["laptop", "hp", "tablet"]
produk_fashion = ["baju", "celana", "sepatu"]
produk_makanan = ["roti", "susu", "keju"]
return chain(produk_elektronik, produk_fashion, produk_makanan)
for produk in ambil_semua_produk():
print(produk)
islice — Lazy Slicing of Iterables
#
islice takes a subset of elements from an iterable without loading all the data first. Useful for generators or very large data streams.
from itertools import islice
# Take the first N elements from a generator
def angka_tak_terhingga():
n = 0
while True:
yield n
n += 1
# ANTI-PATTERN: you can't slice a regular generator
# gen = angka_tak_terhingga()
# gen[:10] # TypeError: 'generator' object is not subscriptable
# CORRECT: use islice
sepuluh_pertama = list(islice(angka_tak_terhingga(), 10))
print(sepuluh_pertama) # [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
# islice(iterable, start, stop, step)
data = range(100)
print(list(islice(data, 10, 20))) # [10, 11, ..., 19]
print(list(islice(data, 0, 50, 5))) # [0, 5, 10, ..., 45]
# Real example: read a CSV file line by line, skip the header, take 100 lines
def baca_batch(filepath, skip=1, ambil=100):
with open(filepath) as f:
for baris in islice(f, skip, skip + ambil):
yield baris.strip()
groupby — Grouping Consecutive Elements
#
groupby groups consecutive elements that have the same key value. Important: the data must be sorted by the key before grouping.
from itertools import groupby
data = [
{"nama": "Alice", "dept": "Engineering"},
{"nama": "Bob", "dept": "Engineering"},
{"nama": "Carol", "dept": "Marketing"},
{"nama": "Dave", "dept": "Marketing"},
{"nama": "Eve", "dept": "Engineering"}, # Engineering again after Marketing
]
# ANTI-PATTERN: groupby without sorting first
for dept, anggota in groupby(data, key=lambda x: x["dept"]):
print(dept, list(anggota))
# Engineering: Alice, Bob
# Marketing: Carol, Dave
# Engineering: Eve <-- appears again because it wasn't sorted!
# CORRECT: sort first by the same key
data_sorted = sorted(data, key=lambda x: x["dept"])
for dept, anggota in groupby(data_sorted, key=lambda x: x["dept"]):
print(dept, [a["nama"] for a in anggota])
# Engineering: ['Alice', 'Bob', 'Eve']
# Marketing: ['Carol', 'Dave']
from itertools import groupby
# Real example: group transactions by date
transaksi = [
{"tanggal": "2024-01-01", "jumlah": 150000},
{"tanggal": "2024-01-01", "jumlah": 75000},
{"tanggal": "2024-01-02", "jumlah": 200000},
{"tanggal": "2024-01-03", "jumlah": 50000},
{"tanggal": "2024-01-03", "jumlah": 125000},
]
transaksi.sort(key=lambda x: x["tanggal"])
for tanggal, grup in groupby(transaksi, key=lambda x: x["tanggal"]):
total = sum(t["jumlah"] for t in grup)
print(f"{tanggal}: Rp{total:,.0f}")
# 2024-01-01: Rp225,000
# 2024-01-02: Rp200,000
# 2024-01-03: Rp175,000
product, combinations, permutations — Combinatorics
#
from itertools import product, combinations, permutations
# product() -- Cartesian product (like nested for loops)
warna = ["merah", "biru"]
ukuran = ["S", "M", "L"]
for w, u in product(warna, ukuran):
print(f"{w}-{u}", end=" ")
# merah-S merah-M merah-L biru-S biru-M biru-L
# product() with repeat -- card combinations
# all dice pairs (6x6 = 36 possibilities)
dadu = list(product(range(1, 7), repeat=2))
print(len(dadu)) # 36
# combinations() -- combinations without repetition, order doesn't matter
tim = ["Alice", "Bob", "Carol", "Dave"]
for pasangan in combinations(tim, 2):
print(pasangan)
# ('Alice', 'Bob'), ('Alice', 'Carol'), ('Alice', 'Dave'),
# ('Bob', 'Carol'), ('Bob', 'Dave'), ('Carol', 'Dave')
print(len(list(combinations(tim, 2)))) # 6 = C(4,2)
# permutations() -- like combinations but order matters
for urutan in permutations(["A", "B", "C"], 2):
print(urutan)
# ('A', 'B'), ('A', 'C'), ('B', 'A'), ('B', 'C'), ('C', 'A'), ('C', 'B')
print(len(list(permutations(["A", "B", "C"], 2)))) # 6 = P(3,2)
count, cycle, repeat — Infinite Iterators
#
from itertools import count, cycle, repeat
# count(start, step) -- count from start, forever
for i, item in zip(count(1), ["a", "b", "c", "d"]):
print(f"{i}. {item}")
# 1. a 2. b 3. c 4. d
# cycle() -- repeat elements cyclically
warna_alternating = cycle(["merah", "putih"])
for i, warna in zip(range(6), warna_alternating):
print(f"Row {i}: {warna}")
# Row 0: merah Row 1: putih Row 2: merah ...
# repeat(object, times) -- repeat an element n times
print(list(repeat("x", 5))) # ['x', 'x', 'x', 'x', 'x']
# Useful with map() to provide fixed arguments
from itertools import starmap
print(list(starmap(pow, [(2, 3), (3, 2), (4, 2)]))) # [8, 9, 16]
takewhile and dropwhile
#
from itertools import takewhile, dropwhile
data = [1, 3, 5, 2, 8, 4, 7]
# takewhile() -- take elements while the condition is True, stop when False
print(list(takewhile(lambda x: x < 6, data))) # [1, 3, 5]
# stops at the FIRST False condition
print(list(takewhile(lambda x: x % 2 != 0, data))) # [1, 3, 5]
# stops at 2 (the first even number)
# dropwhile() -- skip elements while the condition is True, take the rest
print(list(dropwhile(lambda x: x < 6, data))) # [2, 8, 4, 7]
# starts taking from the first False: 8 < 6 is False, so it starts at 8
data2 = [1, 2, 3, 10, 4, 5]
print(list(dropwhile(lambda x: x < 5, data2))) # [10, 4, 5]
zip_longest and pairwise
#
from itertools import zip_longest, pairwise
# zip_longest() -- zip but doesn't stop at the shortest iterable
nama = ["Alice", "Bob", "Carol"]
skor = [85, 92]
# Regular zip -- stops at the shortest
print(list(zip(nama, skor))) # [('Alice', 85), ('Bob', 92)]
# zip_longest -- fills with fillvalue
print(list(zip_longest(nama, skor, fillvalue=0)))
# [('Alice', 85), ('Bob', 92), ('Carol', 0)]
# pairwise() -- pair consecutive elements (Python 3.10+)
data = [1, 2, 3, 4, 5]
print(list(pairwise(data))) # [(1,2), (2,3), (3,4), (4,5)]
# Real example: compute differences between consecutive data points
harga = [100, 105, 98, 112, 108]
selisih = [b - a for a, b in pairwise(harga)]
print(selisih) # [5, -7, 14, -4]
The functools Module
#
partial — Functions with Locked Arguments
#
partial creates a new function from an existing one with some arguments already fixed. Useful for avoiding repetition of the same arguments.
from functools import partial
# Original function
def kirim_email(to: str, subject: str, body: str, from_addr: str = "[email protected]"):
print(f"From: {from_addr} | To: {to} | Subject: {subject}")
print(f"Body: {body}")
# ANTI-PATTERN: repeating the same arguments over and over
kirim_email("[email protected]", "Selamat Datang", "...", from_addr="[email protected]")
kirim_email("[email protected]", "Selamat Datang", "...", from_addr="[email protected]")
# CORRECT: create a new function with the arguments locked
kirim_dari_support = partial(kirim_email, from_addr="[email protected]")
kirim_dari_support("[email protected]", "Selamat Datang", "...")
kirim_dari_support("[email protected]", "Verifikasi Email", "...")
# Another example: sorting with a configured key
from functools import partial
def ambil_field(obj, field):
return obj[field]
data = [{"nama": "Carol", "usia": 30}, {"nama": "Alice", "usia": 25}, {"nama": "Bob", "usia": 28}]
ambil_nama = partial(ambil_field, field="nama")
ambil_usia = partial(ambil_field, field="usia")
print(sorted(data, key=ambil_nama)) # sort by name
print(sorted(data, key=ambil_usia)) # sort by age
lru_cache — Automatic Memoization
#
lru_cache (Least Recently Used cache) stores function call results so calls with the same arguments don’t need to be recomputed.
from functools import lru_cache
import time
# Without a cache -- very slow for large n
def fibonacci_lambat(n):
if n < 2:
return n
return fibonacci_lambat(n - 1) + fibonacci_lambat(n - 2)
# With a cache -- very fast
@lru_cache(maxsize=128)
def fibonacci(n):
if n < 2:
return n
return fibonacci(n - 1) + fibonacci(n - 2)
print(fibonacci(100)) # instant
print(fibonacci.cache_info())
# CacheInfo(hits=98, misses=101, maxsize=128, currsize=101)
# cache_clear() -- empty the cache
fibonacci.cache_clear()
# @cache (Python 3.9+) -- like lru_cache(maxsize=None), no limit
from functools import cache
@cache
def faktorial(n):
return 1 if n == 0 else n * faktorial(n - 1)
from functools import lru_cache
# Real example: caching expensive query results
@lru_cache(maxsize=256)
def ambil_data_kota(kota_id: int) -> dict:
"""Simulate a slow database query."""
time.sleep(0.1) # simulate database latency
return {"id": kota_id, "nama": f"Kota-{kota_id}", "populasi": kota_id * 10000}
# First call: slow (database query)
data = ambil_data_kota(1) # 0.1 seconds
# Second call with the same argument: instant (from cache)
data = ambil_data_kota(1) # < 1ms
lru_cache only works for functions with hashable (immutable) arguments. Functions accepting lists, dicts, or mutable objects as arguments can’t be cached directly. Convert to tuples or frozensets first if needed.reduce — Accumulating Values
#
reduce applies a two-argument function cumulatively to iterable elements from left to right.
from functools import reduce
angka = [1, 2, 3, 4, 5]
# ANTI-PATTERN: use reduce for operations that already have built-ins
total = reduce(lambda a, b: a + b, angka) # use sum() instead!
maksimum = reduce(lambda a, b: a if a > b else b, angka) # use max() instead!
# CORRECT: use reduce for operations without built-ins
# Example: nested dict access
from functools import reduce
config = {
"database": {
"primary": {
"host": "db.example.com",
"port": 5432
}
}
}
def ambil_nested(data: dict, keys: list):
"""Get a value from a nested dict with a list of keys."""
return reduce(lambda d, k: d[k], keys, data)
print(ambil_nested(config, ["database", "primary", "host"])) # "db.example.com"
print(ambil_nested(config, ["database", "primary", "port"])) # 5432
# Another example: transformation pipeline
operasi = [
lambda x: x * 2,
lambda x: x + 10,
lambda x: x ** 2,
]
hasil = reduce(lambda val, fn: fn(val), operasi, 5)
# 5 -> *2 -> 10 -> +10 -> 20 -> **2 -> 400
print(hasil) # 400
wraps — Proper Decorators
#
When creating decorators, use @wraps so the decorated function retains its original metadata (name, docstring, signature).
from functools import wraps
import time
# ANTI-PATTERN: decorator without @wraps
def timer_buruk(func):
def wrapper(*args, **kwargs):
start = time.time()
result = func(*args, **kwargs)
print(f"Waktu: {time.time() - start:.3f}s")
return result
return wrapper
@timer_buruk
def hitung():
"""Fungsi penghitungan."""
return sum(range(1000000))
print(hitung.__name__) # "wrapper" -- original name lost!
print(hitung.__doc__) # None -- docstring lost!
# CORRECT: use @wraps
def timer(func):
@wraps(func)
def wrapper(*args, **kwargs):
start = time.time()
result = func(*args, **kwargs)
print(f"Waktu: {time.time() - start:.3f}s")
return result
return wrapper
@timer
def hitung():
"""Fungsi penghitungan."""
return sum(range(1000000))
print(hitung.__name__) # "hitung" -- name preserved
print(hitung.__doc__) # "Fungsi penghitungan." -- docstring preserved
total_ordering — Completing Comparison Operators
#
total_ordering fills in the missing comparison operators for a class. You only need to define __eq__ and one of __lt__, __le__, __gt__, or __ge__.
from functools import total_ordering
@total_ordering
class Mahasiswa:
def __init__(self, nama: str, ipk: float):
self.nama = nama
self.ipk = ipk
def __eq__(self, other):
return self.ipk == other.ipk
def __lt__(self, other):
return self.ipk < other.ipk
# total_ordering automatically fills in: >, >=, <=
mhs1 = Mahasiswa("Alice", 3.75)
mhs2 = Mahasiswa("Bob", 3.50)
print(mhs1 > mhs2) # True
print(mhs1 >= mhs2) # True
print(mhs1 <= mhs2) # False
print(sorted([mhs1, mhs2])) # [Bob(3.5), Alice(3.75)]
Combining itertools and functools #
from itertools import groupby, chain
from functools import reduce
# Example: sales analysis per category from several data sources
penjualan_jan = [
{"kategori": "Elektronik", "total": 5000000},
{"kategori": "Fashion", "total": 2000000},
{"kategori": "Elektronik", "total": 3500000},
]
penjualan_feb = [
{"kategori": "Fashion", "total": 2500000},
{"kategori": "Elektronik", "total": 4000000},
{"kategori": "Makanan", "total": 1500000},
]
# Combine all data with chain
semua = sorted(
chain(penjualan_jan, penjualan_feb),
key=lambda x: x["kategori"]
)
# Group and sum per category
for kategori, grup in groupby(semua, key=lambda x: x["kategori"]):
total = reduce(lambda acc, x: acc + x["total"], grup, 0)
print(f"{kategori}: Rp{total:,.0f}")
# Elektronik: Rp12,500,000
# Fashion: Rp4,500,000
# Makanan: Rp1,500,000
Summary #
chainfor combining several iterables without copying to memory;chain.from_iterablefor flattening lists of lists.islicefor taking a subset of elements from a generator or large iterable without loading all the data.groupbyfor grouping elements — must sort first by the same key, or the results won’t match expectations.productfor the Cartesian product (replacing nested for loops);combinationsfor unordered combinations;permutationsfor ordered combinations.partialfor creating a new function with some arguments locked — avoids repeating the same arguments.lru_cache/cachefor automatic memoization — good for pure functions frequently called with the same arguments; arguments must be hashable.reducefor accumulating values without a built-in — avoid it for operations that already havesum(),max(),min().@wrapsis required in every decorator so the original function’s metadata isn’t lost.